2024-11-19 11:14:08,877 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 11:14:08,890 main DEBUG Took 0.011202 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-19 11:14:08,890 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-19 11:14:08,891 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-19 11:14:08,892 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-19 11:14:08,894 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:14:08,901 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-19 11:14:08,914 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:14:08,916 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:14:08,917 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:14:08,917 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:14:08,918 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:14:08,918 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:14:08,920 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:14:08,920 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:14:08,921 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:14:08,921 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:14:08,923 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:14:08,923 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:14:08,924 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:14:08,924 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:14:08,924 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:14:08,925 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:14:08,925 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:14:08,926 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:14:08,926 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:14:08,927 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:14:08,927 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:14:08,928 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:14:08,928 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:14:08,929 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:14:08,929 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:14:08,930 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-19 11:14:08,932 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:14:08,933 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-19 11:14:08,936 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-19 11:14:08,936 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-19 11:14:08,938 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-19 11:14:08,938 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-19 11:14:08,945 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-19 11:14:08,948 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-19 11:14:08,950 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-19 11:14:08,950 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-19 11:14:08,950 main DEBUG createAppenders(={Console}) 2024-11-19 11:14:08,951 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-19 11:14:08,951 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 11:14:08,952 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-19 11:14:08,952 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-19 11:14:08,953 main DEBUG OutputStream closed 2024-11-19 11:14:08,953 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-19 11:14:08,953 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-19 11:14:08,953 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-19 11:14:09,028 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-19 11:14:09,031 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-19 11:14:09,032 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-19 11:14:09,033 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-19 11:14:09,034 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-19 11:14:09,034 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-19 11:14:09,034 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-19 11:14:09,035 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-19 11:14:09,035 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-19 11:14:09,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-19 11:14:09,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-19 11:14:09,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-19 11:14:09,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-19 11:14:09,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-19 11:14:09,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-19 11:14:09,038 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-19 11:14:09,038 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-19 11:14:09,039 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-19 11:14:09,041 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19 11:14:09,042 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-19 11:14:09,042 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-19 11:14:09,043 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-19T11:14:09,253 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b 2024-11-19 11:14:09,256 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-19 11:14:09,256 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19T11:14:09,265 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-19T11:14:09,287 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-19T11:14:09,308 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/cluster_5de41c1e-6874-2d94-bb17-966f646923cf, deleteOnExit=true 2024-11-19T11:14:09,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-19T11:14:09,310 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/test.cache.data in system properties and HBase conf 2024-11-19T11:14:09,311 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/hadoop.tmp.dir in system properties and HBase conf 2024-11-19T11:14:09,312 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/hadoop.log.dir in system properties and HBase conf 2024-11-19T11:14:09,313 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-19T11:14:09,313 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-19T11:14:09,314 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-19T11:14:09,394 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-19T11:14:09,479 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T11:14:09,482 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-19T11:14:09,483 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-19T11:14:09,483 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-19T11:14:09,484 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T11:14:09,484 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-19T11:14:09,485 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-19T11:14:09,485 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T11:14:09,486 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T11:14:09,486 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-19T11:14:09,486 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/nfs.dump.dir in system properties and HBase conf 2024-11-19T11:14:09,487 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/java.io.tmpdir in system properties and HBase conf 2024-11-19T11:14:09,487 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T11:14:09,488 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-19T11:14:09,488 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-19T11:14:10,584 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-19T11:14:10,654 INFO [Time-limited test {}] log.Log(170): Logging initialized @2461ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-19T11:14:10,718 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T11:14:10,778 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T11:14:10,798 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T11:14:10,798 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T11:14:10,799 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T11:14:10,810 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T11:14:10,813 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/hadoop.log.dir/,AVAILABLE} 2024-11-19T11:14:10,814 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T11:14:10,992 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/java.io.tmpdir/jetty-localhost-35885-hadoop-hdfs-3_4_1-tests_jar-_-any-11565834455299728691/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T11:14:11,000 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:35885} 2024-11-19T11:14:11,000 INFO [Time-limited test {}] server.Server(415): Started @2808ms 2024-11-19T11:14:11,684 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T11:14:11,710 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T11:14:11,715 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T11:14:11,716 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T11:14:11,716 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-19T11:14:11,721 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/hadoop.log.dir/,AVAILABLE} 2024-11-19T11:14:11,722 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T11:14:11,854 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c68b1bf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/java.io.tmpdir/jetty-localhost-35795-hadoop-hdfs-3_4_1-tests_jar-_-any-9435763718976923548/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T11:14:11,854 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:35795} 2024-11-19T11:14:11,855 INFO [Time-limited test {}] server.Server(415): Started @3662ms 2024-11-19T11:14:11,904 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-19T11:14:13,402 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/cluster_5de41c1e-6874-2d94-bb17-966f646923cf/data/data1/current/BP-1532712323-172.17.0.2-1732014850042/current, will proceed with Du for space computation calculation, 2024-11-19T11:14:13,402 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/cluster_5de41c1e-6874-2d94-bb17-966f646923cf/data/data2/current/BP-1532712323-172.17.0.2-1732014850042/current, will proceed with Du for space computation calculation, 2024-11-19T11:14:13,440 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-19T11:14:13,488 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x66d71c69f694af92 with lease ID 0x1ddddf0a0c25391e: Processing first storage report for DS-d5ea183d-af01-456d-aa39-9af86c1d492a from datanode DatanodeRegistration(127.0.0.1:42293, datanodeUuid=9754eb23-22ad-4701-a420-3678696778b6, infoPort=36119, infoSecurePort=0, ipcPort=43297, storageInfo=lv=-57;cid=testClusterID;nsid=476841165;c=1732014850042) 2024-11-19T11:14:13,490 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x66d71c69f694af92 with lease ID 0x1ddddf0a0c25391e: from storage DS-d5ea183d-af01-456d-aa39-9af86c1d492a node DatanodeRegistration(127.0.0.1:42293, datanodeUuid=9754eb23-22ad-4701-a420-3678696778b6, infoPort=36119, infoSecurePort=0, ipcPort=43297, storageInfo=lv=-57;cid=testClusterID;nsid=476841165;c=1732014850042), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-19T11:14:13,490 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x66d71c69f694af92 with lease ID 0x1ddddf0a0c25391e: Processing first storage report for DS-958441bb-e8d2-42d7-83e3-5541df3efeab from datanode DatanodeRegistration(127.0.0.1:42293, datanodeUuid=9754eb23-22ad-4701-a420-3678696778b6, infoPort=36119, infoSecurePort=0, ipcPort=43297, storageInfo=lv=-57;cid=testClusterID;nsid=476841165;c=1732014850042) 2024-11-19T11:14:13,490 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x66d71c69f694af92 with lease ID 0x1ddddf0a0c25391e: from storage DS-958441bb-e8d2-42d7-83e3-5541df3efeab node DatanodeRegistration(127.0.0.1:42293, datanodeUuid=9754eb23-22ad-4701-a420-3678696778b6, infoPort=36119, infoSecurePort=0, ipcPort=43297, storageInfo=lv=-57;cid=testClusterID;nsid=476841165;c=1732014850042), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-19T11:14:13,569 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b 2024-11-19T11:14:13,637 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/cluster_5de41c1e-6874-2d94-bb17-966f646923cf/zookeeper_0, clientPort=58397, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/cluster_5de41c1e-6874-2d94-bb17-966f646923cf/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/cluster_5de41c1e-6874-2d94-bb17-966f646923cf/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-19T11:14:13,647 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58397 2024-11-19T11:14:13,657 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T11:14:13,660 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T11:14:13,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741825_1001 (size=7) 2024-11-19T11:14:14,293 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094 with version=8 2024-11-19T11:14:14,293 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/hbase-staging 2024-11-19T11:14:14,378 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-19T11:14:14,577 INFO [Time-limited test {}] client.ConnectionUtils(128): master/16c88966c0af:0 server-side Connection retries=45 2024-11-19T11:14:14,586 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T11:14:14,586 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T11:14:14,591 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T11:14:14,591 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T11:14:14,591 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T11:14:14,711 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-19T11:14:14,762 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-19T11:14:14,770 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-19T11:14:14,774 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T11:14:14,797 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4680 (auto-detected) 2024-11-19T11:14:14,798 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-19T11:14:14,816 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41441 2024-11-19T11:14:14,836 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41441 connecting to ZooKeeper ensemble=127.0.0.1:58397 2024-11-19T11:14:14,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:414410x0, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T11:14:14,967 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41441-0x10152cd49a00000 connected 2024-11-19T11:14:15,521 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T11:14:15,523 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T11:14:15,531 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T11:14:15,534 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094, hbase.cluster.distributed=false 2024-11-19T11:14:15,554 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T11:14:15,558 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41441 2024-11-19T11:14:15,559 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41441 2024-11-19T11:14:15,559 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41441 2024-11-19T11:14:15,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41441 2024-11-19T11:14:15,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41441 2024-11-19T11:14:15,675 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/16c88966c0af:0 server-side Connection retries=45 2024-11-19T11:14:15,677 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T11:14:15,677 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T11:14:15,677 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T11:14:15,678 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T11:14:15,678 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T11:14:15,681 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T11:14:15,683 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T11:14:15,685 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35833 2024-11-19T11:14:15,687 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35833 connecting to ZooKeeper ensemble=127.0.0.1:58397 2024-11-19T11:14:15,688 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T11:14:15,691 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T11:14:15,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:358330x0, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T11:14:15,711 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:358330x0, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T11:14:15,711 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35833-0x10152cd49a00001 connected 2024-11-19T11:14:15,714 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T11:14:15,722 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T11:14:15,725 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35833-0x10152cd49a00001, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T11:14:15,730 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35833-0x10152cd49a00001, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T11:14:15,731 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35833 2024-11-19T11:14:15,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35833 2024-11-19T11:14:15,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35833 2024-11-19T11:14:15,734 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35833 2024-11-19T11:14:15,734 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35833 2024-11-19T11:14:15,751 DEBUG [M:0;16c88966c0af:41441 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;16c88966c0af:41441 2024-11-19T11:14:15,752 INFO [master/16c88966c0af:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/16c88966c0af,41441,1732014854424 2024-11-19T11:14:15,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10152cd49a00001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T11:14:15,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T11:14:15,786 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/16c88966c0af,41441,1732014854424 2024-11-19T11:14:15,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T11:14:15,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10152cd49a00001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T11:14:15,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10152cd49a00001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T11:14:15,861 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T11:14:15,862 INFO [master/16c88966c0af:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/16c88966c0af,41441,1732014854424 from backup master directory 2024-11-19T11:14:15,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10152cd49a00001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T11:14:15,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/16c88966c0af,41441,1732014854424 2024-11-19T11:14:15,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T11:14:15,890 WARN [master/16c88966c0af:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T11:14:15,890 INFO [master/16c88966c0af:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=16c88966c0af,41441,1732014854424 2024-11-19T11:14:15,892 INFO [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-19T11:14:15,894 INFO [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-19T11:14:15,948 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/hbase.id] with ID: 8b4a9e94-f011-4c02-bbc9-3820ab93d5c2 2024-11-19T11:14:15,948 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/.tmp/hbase.id 2024-11-19T11:14:15,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741826_1002 (size=42) 2024-11-19T11:14:15,959 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/.tmp/hbase.id]:[hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/hbase.id] 2024-11-19T11:14:16,006 INFO [master/16c88966c0af:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T11:14:16,012 INFO [master/16c88966c0af:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-19T11:14:16,031 INFO [master/16c88966c0af:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-19T11:14:16,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10152cd49a00001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T11:14:16,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T11:14:16,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741827_1003 (size=196) 2024-11-19T11:14:16,077 INFO [master/16c88966c0af:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:14:16,080 INFO [master/16c88966c0af:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-19T11:14:16,095 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:16,099 INFO [master/16c88966c0af:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T11:14:16,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741828_1004 (size=1189) 2024-11-19T11:14:16,546 INFO [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/data/master/store 2024-11-19T11:14:16,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741829_1005 (size=34) 2024-11-19T11:14:16,574 INFO [master/16c88966c0af:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-19T11:14:16,577 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:14:16,578 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T11:14:16,578 INFO [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T11:14:16,579 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T11:14:16,580 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T11:14:16,580 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T11:14:16,580 INFO [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T11:14:16,581 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732014856578Disabling compacts and flushes for region at 1732014856578Disabling writes for close at 1732014856580 (+2 ms)Writing region close event to WAL at 1732014856580Closed at 1732014856580 2024-11-19T11:14:16,584 WARN [master/16c88966c0af:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/data/master/store/.initializing 2024-11-19T11:14:16,584 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/WALs/16c88966c0af,41441,1732014854424 2024-11-19T11:14:16,594 INFO [master/16c88966c0af:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T11:14:16,614 INFO [master/16c88966c0af:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=16c88966c0af%2C41441%2C1732014854424, suffix=, logDir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/WALs/16c88966c0af,41441,1732014854424, archiveDir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/oldWALs, maxLogs=10 2024-11-19T11:14:16,650 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/WALs/16c88966c0af,41441,1732014854424/16c88966c0af%2C41441%2C1732014854424.1732014856621, exclude list is [], retry=0 2024-11-19T11:14:16,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42293,DS-d5ea183d-af01-456d-aa39-9af86c1d492a,DISK] 2024-11-19T11:14:16,674 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-19T11:14:16,704 INFO [master/16c88966c0af:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/WALs/16c88966c0af,41441,1732014854424/16c88966c0af%2C41441%2C1732014854424.1732014856621 2024-11-19T11:14:16,705 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36119:36119)] 2024-11-19T11:14:16,706 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-19T11:14:16,706 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:14:16,709 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:14:16,710 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:14:16,745 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:14:16,770 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-19T11:14:16,773 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:16,776 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T11:14:16,776 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:14:16,781 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-19T11:14:16,781 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:16,783 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:14:16,783 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:14:16,787 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-19T11:14:16,788 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:16,789 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:14:16,790 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:14:16,793 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-19T11:14:16,793 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:16,795 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:14:16,795 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:14:16,799 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:14:16,801 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:14:16,805 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:14:16,806 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:14:16,809 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T11:14:16,813 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:14:16,817 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T11:14:16,818 INFO [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65285127, jitterRate=-0.027175799012184143}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T11:14:16,824 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732014856721Initializing all the Stores at 1732014856723 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732014856723Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014856724 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014856724Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014856724Cleaning up temporary data from old regions at 1732014856806 (+82 ms)Region opened successfully at 1732014856824 (+18 ms) 2024-11-19T11:14:16,826 INFO [master/16c88966c0af:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-19T11:14:16,856 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4be0e44b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=16c88966c0af/172.17.0.2:0 2024-11-19T11:14:16,884 INFO [master/16c88966c0af:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-19T11:14:16,893 INFO [master/16c88966c0af:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-19T11:14:16,893 INFO [master/16c88966c0af:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-19T11:14:16,895 INFO [master/16c88966c0af:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T11:14:16,897 INFO [master/16c88966c0af:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-19T11:14:16,902 INFO [master/16c88966c0af:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-19T11:14:16,902 INFO [master/16c88966c0af:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-19T11:14:16,926 INFO [master/16c88966c0af:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-19T11:14:16,937 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-19T11:14:16,962 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-19T11:14:16,965 INFO [master/16c88966c0af:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-19T11:14:16,967 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-19T11:14:16,983 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-19T11:14:16,985 INFO [master/16c88966c0af:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-19T11:14:16,988 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-19T11:14:16,994 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-19T11:14:16,995 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-19T11:14:17,004 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-19T11:14:17,024 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-19T11:14:17,036 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-19T11:14:17,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10152cd49a00001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T11:14:17,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T11:14:17,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10152cd49a00001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T11:14:17,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T11:14:17,051 INFO [master/16c88966c0af:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=16c88966c0af,41441,1732014854424, sessionid=0x10152cd49a00000, setting cluster-up flag (Was=false) 2024-11-19T11:14:17,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10152cd49a00001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T11:14:17,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T11:14:17,110 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-19T11:14:17,113 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=16c88966c0af,41441,1732014854424 2024-11-19T11:14:17,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T11:14:17,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10152cd49a00001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T11:14:17,162 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-19T11:14:17,166 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=16c88966c0af,41441,1732014854424 2024-11-19T11:14:17,175 INFO [master/16c88966c0af:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-19T11:14:17,240 INFO [RS:0;16c88966c0af:35833 {}] regionserver.HRegionServer(746): ClusterId : 8b4a9e94-f011-4c02-bbc9-3820ab93d5c2 2024-11-19T11:14:17,243 DEBUG [RS:0;16c88966c0af:35833 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T11:14:17,249 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T11:14:17,257 INFO [master/16c88966c0af:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-19T11:14:17,259 DEBUG [RS:0;16c88966c0af:35833 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T11:14:17,259 DEBUG [RS:0;16c88966c0af:35833 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T11:14:17,263 INFO [master/16c88966c0af:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-19T11:14:17,269 DEBUG [RS:0;16c88966c0af:35833 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T11:14:17,269 DEBUG [RS:0;16c88966c0af:35833 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a863580, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=16c88966c0af/172.17.0.2:0 2024-11-19T11:14:17,267 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 16c88966c0af,41441,1732014854424 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-19T11:14:17,274 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/16c88966c0af:0, corePoolSize=5, maxPoolSize=5 2024-11-19T11:14:17,274 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/16c88966c0af:0, corePoolSize=5, maxPoolSize=5 2024-11-19T11:14:17,274 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/16c88966c0af:0, corePoolSize=5, maxPoolSize=5 2024-11-19T11:14:17,274 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/16c88966c0af:0, corePoolSize=5, maxPoolSize=5 2024-11-19T11:14:17,274 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/16c88966c0af:0, corePoolSize=10, maxPoolSize=10 2024-11-19T11:14:17,274 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/16c88966c0af:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:14:17,275 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/16c88966c0af:0, corePoolSize=2, maxPoolSize=2 2024-11-19T11:14:17,275 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/16c88966c0af:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:14:17,276 INFO [master/16c88966c0af:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732014887276 2024-11-19T11:14:17,278 INFO [master/16c88966c0af:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-19T11:14:17,279 INFO [master/16c88966c0af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-19T11:14:17,280 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T11:14:17,280 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-19T11:14:17,283 DEBUG [RS:0;16c88966c0af:35833 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;16c88966c0af:35833 2024-11-19T11:14:17,284 INFO [master/16c88966c0af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-19T11:14:17,284 INFO [master/16c88966c0af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-19T11:14:17,285 INFO [master/16c88966c0af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-19T11:14:17,285 INFO [master/16c88966c0af:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-19T11:14:17,286 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:17,286 INFO [RS:0;16c88966c0af:35833 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T11:14:17,286 INFO [RS:0;16c88966c0af:35833 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T11:14:17,286 DEBUG [RS:0;16c88966c0af:35833 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T11:14:17,286 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-19T11:14:17,289 INFO [RS:0;16c88966c0af:35833 {}] regionserver.HRegionServer(2659): reportForDuty to master=16c88966c0af,41441,1732014854424 with port=35833, startcode=1732014855640 2024-11-19T11:14:17,287 INFO [master/16c88966c0af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T11:14:17,294 INFO [master/16c88966c0af:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-19T11:14:17,295 INFO [master/16c88966c0af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-19T11:14:17,295 INFO [master/16c88966c0af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-19T11:14:17,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741831_1007 (size=1321) 2024-11-19T11:14:17,297 INFO [master/16c88966c0af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-19T11:14:17,298 INFO [master/16c88966c0af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-19T11:14:17,299 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/16c88966c0af:0:becomeActiveMaster-HFileCleaner.large.0-1732014857299,5,FailOnTimeoutGroup] 2024-11-19T11:14:17,300 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/16c88966c0af:0:becomeActiveMaster-HFileCleaner.small.0-1732014857299,5,FailOnTimeoutGroup] 2024-11-19T11:14:17,300 INFO [master/16c88966c0af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T11:14:17,300 INFO [master/16c88966c0af:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-19T11:14:17,301 INFO [master/16c88966c0af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-19T11:14:17,301 INFO [master/16c88966c0af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-19T11:14:17,306 DEBUG [RS:0;16c88966c0af:35833 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T11:14:17,338 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57771, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T11:14:17,343 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41441 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 16c88966c0af,35833,1732014855640 2024-11-19T11:14:17,345 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41441 {}] master.ServerManager(517): Registering regionserver=16c88966c0af,35833,1732014855640 2024-11-19T11:14:17,357 DEBUG [RS:0;16c88966c0af:35833 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094 2024-11-19T11:14:17,357 DEBUG [RS:0;16c88966c0af:35833 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37775 2024-11-19T11:14:17,357 DEBUG [RS:0;16c88966c0af:35833 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T11:14:17,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T11:14:17,373 DEBUG [RS:0;16c88966c0af:35833 {}] zookeeper.ZKUtil(111): regionserver:35833-0x10152cd49a00001, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/16c88966c0af,35833,1732014855640 2024-11-19T11:14:17,374 WARN [RS:0;16c88966c0af:35833 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T11:14:17,374 INFO [RS:0;16c88966c0af:35833 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T11:14:17,374 DEBUG [RS:0;16c88966c0af:35833 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/WALs/16c88966c0af,35833,1732014855640 2024-11-19T11:14:17,376 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [16c88966c0af,35833,1732014855640] 2024-11-19T11:14:17,396 INFO [RS:0;16c88966c0af:35833 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T11:14:17,412 INFO [RS:0;16c88966c0af:35833 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T11:14:17,416 INFO [RS:0;16c88966c0af:35833 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T11:14:17,417 INFO [RS:0;16c88966c0af:35833 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T11:14:17,417 INFO [RS:0;16c88966c0af:35833 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T11:14:17,422 INFO [RS:0;16c88966c0af:35833 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T11:14:17,423 INFO [RS:0;16c88966c0af:35833 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T11:14:17,424 DEBUG [RS:0;16c88966c0af:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/16c88966c0af:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:14:17,424 DEBUG [RS:0;16c88966c0af:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/16c88966c0af:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:14:17,424 DEBUG [RS:0;16c88966c0af:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/16c88966c0af:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:14:17,424 DEBUG [RS:0;16c88966c0af:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/16c88966c0af:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:14:17,425 DEBUG [RS:0;16c88966c0af:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/16c88966c0af:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:14:17,425 DEBUG [RS:0;16c88966c0af:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/16c88966c0af:0, corePoolSize=2, maxPoolSize=2 2024-11-19T11:14:17,425 DEBUG [RS:0;16c88966c0af:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/16c88966c0af:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:14:17,425 DEBUG [RS:0;16c88966c0af:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/16c88966c0af:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:14:17,425 DEBUG [RS:0;16c88966c0af:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/16c88966c0af:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:14:17,425 DEBUG [RS:0;16c88966c0af:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/16c88966c0af:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:14:17,426 DEBUG [RS:0;16c88966c0af:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/16c88966c0af:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:14:17,426 DEBUG [RS:0;16c88966c0af:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/16c88966c0af:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:14:17,426 DEBUG [RS:0;16c88966c0af:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/16c88966c0af:0, corePoolSize=3, maxPoolSize=3 2024-11-19T11:14:17,426 DEBUG [RS:0;16c88966c0af:35833 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0, corePoolSize=3, maxPoolSize=3 2024-11-19T11:14:17,427 INFO [RS:0;16c88966c0af:35833 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T11:14:17,428 INFO [RS:0;16c88966c0af:35833 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T11:14:17,428 INFO [RS:0;16c88966c0af:35833 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T11:14:17,428 INFO [RS:0;16c88966c0af:35833 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T11:14:17,428 INFO [RS:0;16c88966c0af:35833 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T11:14:17,428 INFO [RS:0;16c88966c0af:35833 {}] hbase.ChoreService(168): Chore ScheduledChore name=16c88966c0af,35833,1732014855640-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T11:14:17,451 INFO [RS:0;16c88966c0af:35833 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T11:14:17,452 INFO [RS:0;16c88966c0af:35833 {}] hbase.ChoreService(168): Chore ScheduledChore name=16c88966c0af,35833,1732014855640-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T11:14:17,452 INFO [RS:0;16c88966c0af:35833 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T11:14:17,453 INFO [RS:0;16c88966c0af:35833 {}] regionserver.Replication(171): 16c88966c0af,35833,1732014855640 started 2024-11-19T11:14:17,470 INFO [RS:0;16c88966c0af:35833 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T11:14:17,470 INFO [RS:0;16c88966c0af:35833 {}] regionserver.HRegionServer(1482): Serving as 16c88966c0af,35833,1732014855640, RpcServer on 16c88966c0af/172.17.0.2:35833, sessionid=0x10152cd49a00001 2024-11-19T11:14:17,471 DEBUG [RS:0;16c88966c0af:35833 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T11:14:17,471 DEBUG [RS:0;16c88966c0af:35833 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 16c88966c0af,35833,1732014855640 2024-11-19T11:14:17,471 DEBUG [RS:0;16c88966c0af:35833 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16c88966c0af,35833,1732014855640' 2024-11-19T11:14:17,471 DEBUG [RS:0;16c88966c0af:35833 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T11:14:17,472 DEBUG [RS:0;16c88966c0af:35833 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T11:14:17,473 DEBUG [RS:0;16c88966c0af:35833 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T11:14:17,473 DEBUG [RS:0;16c88966c0af:35833 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T11:14:17,473 DEBUG [RS:0;16c88966c0af:35833 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 16c88966c0af,35833,1732014855640 2024-11-19T11:14:17,473 DEBUG [RS:0;16c88966c0af:35833 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16c88966c0af,35833,1732014855640' 2024-11-19T11:14:17,473 DEBUG [RS:0;16c88966c0af:35833 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T11:14:17,474 DEBUG [RS:0;16c88966c0af:35833 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T11:14:17,475 DEBUG [RS:0;16c88966c0af:35833 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T11:14:17,475 INFO [RS:0;16c88966c0af:35833 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T11:14:17,475 INFO [RS:0;16c88966c0af:35833 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T11:14:17,585 INFO [RS:0;16c88966c0af:35833 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T11:14:17,590 INFO [RS:0;16c88966c0af:35833 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=16c88966c0af%2C35833%2C1732014855640, suffix=, logDir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/WALs/16c88966c0af,35833,1732014855640, archiveDir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/oldWALs, maxLogs=32 2024-11-19T11:14:17,606 DEBUG [RS:0;16c88966c0af:35833 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/WALs/16c88966c0af,35833,1732014855640/16c88966c0af%2C35833%2C1732014855640.1732014857593, exclude list is [], retry=0 2024-11-19T11:14:17,610 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42293,DS-d5ea183d-af01-456d-aa39-9af86c1d492a,DISK] 2024-11-19T11:14:17,614 INFO [RS:0;16c88966c0af:35833 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/WALs/16c88966c0af,35833,1732014855640/16c88966c0af%2C35833%2C1732014855640.1732014857593 2024-11-19T11:14:17,614 DEBUG [RS:0;16c88966c0af:35833 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36119:36119)] 2024-11-19T11:14:17,703 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-19T11:14:17,704 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094 2024-11-19T11:14:17,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741833_1009 (size=32) 2024-11-19T11:14:18,122 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:14:18,126 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T11:14:18,130 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T11:14:18,130 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:18,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T11:14:18,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T11:14:18,134 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T11:14:18,135 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:18,135 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T11:14:18,136 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T11:14:18,138 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T11:14:18,138 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:18,139 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T11:14:18,139 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T11:14:18,141 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T11:14:18,142 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:18,142 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T11:14:18,143 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T11:14:18,144 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740 2024-11-19T11:14:18,145 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740 2024-11-19T11:14:18,148 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T11:14:18,148 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T11:14:18,149 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T11:14:18,151 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T11:14:18,154 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T11:14:18,155 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62438426, jitterRate=-0.06959494948387146}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T11:14:18,158 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732014858122Initializing all the Stores at 1732014858125 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732014858125Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732014858125Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014858125Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732014858126 (+1 ms)Cleaning up temporary data from old regions at 1732014858148 (+22 ms)Region opened successfully at 1732014858158 (+10 ms) 2024-11-19T11:14:18,158 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T11:14:18,158 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T11:14:18,158 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T11:14:18,158 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T11:14:18,158 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T11:14:18,159 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T11:14:18,160 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732014858158Disabling compacts and flushes for region at 1732014858158Disabling writes for close at 1732014858158Writing region close event to WAL at 1732014858159 (+1 ms)Closed at 1732014858159 2024-11-19T11:14:18,163 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T11:14:18,163 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-19T11:14:18,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T11:14:18,175 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T11:14:18,178 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-19T11:14:18,335 DEBUG [16c88966c0af:41441 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-19T11:14:18,348 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:14:18,355 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16c88966c0af,35833,1732014855640, state=OPENING 2024-11-19T11:14:18,467 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-19T11:14:18,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10152cd49a00001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T11:14:18,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T11:14:18,479 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T11:14:18,480 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T11:14:18,482 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T11:14:18,484 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=16c88966c0af,35833,1732014855640}] 2024-11-19T11:14:18,664 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T11:14:18,667 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45105, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T11:14:18,677 INFO [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T11:14:18,677 INFO [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T11:14:18,678 INFO [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T11:14:18,681 INFO [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=16c88966c0af%2C35833%2C1732014855640.meta, suffix=.meta, logDir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/WALs/16c88966c0af,35833,1732014855640, archiveDir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/oldWALs, maxLogs=32 2024-11-19T11:14:18,696 DEBUG [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/WALs/16c88966c0af,35833,1732014855640/16c88966c0af%2C35833%2C1732014855640.meta.1732014858683.meta, exclude list is [], retry=0 2024-11-19T11:14:18,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42293,DS-d5ea183d-af01-456d-aa39-9af86c1d492a,DISK] 2024-11-19T11:14:18,703 INFO [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/WALs/16c88966c0af,35833,1732014855640/16c88966c0af%2C35833%2C1732014855640.meta.1732014858683.meta 2024-11-19T11:14:18,703 DEBUG [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36119:36119)] 2024-11-19T11:14:18,704 DEBUG [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T11:14:18,705 DEBUG [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T11:14:18,708 DEBUG [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T11:14:18,712 INFO [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T11:14:18,716 DEBUG [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T11:14:18,717 DEBUG [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:14:18,717 DEBUG [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T11:14:18,717 DEBUG [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T11:14:18,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T11:14:18,722 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T11:14:18,722 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:18,723 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T11:14:18,723 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T11:14:18,724 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T11:14:18,724 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:18,725 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T11:14:18,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T11:14:18,727 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T11:14:18,727 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:18,728 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T11:14:18,728 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T11:14:18,730 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T11:14:18,730 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:18,731 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T11:14:18,731 DEBUG [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T11:14:18,732 DEBUG [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740 2024-11-19T11:14:18,735 DEBUG [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740 2024-11-19T11:14:18,738 DEBUG [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T11:14:18,738 DEBUG [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T11:14:18,739 DEBUG [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T11:14:18,741 DEBUG [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T11:14:18,743 INFO [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72691170, jitterRate=0.08318284153938293}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T11:14:18,744 DEBUG [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T11:14:18,746 DEBUG [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732014858718Writing region info on filesystem at 1732014858718Initializing all the Stores at 1732014858720 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732014858720Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732014858720Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014858720Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732014858720Cleaning up temporary data from old regions at 1732014858738 (+18 ms)Running coprocessor post-open hooks at 1732014858744 (+6 ms)Region opened successfully at 1732014858746 (+2 ms) 2024-11-19T11:14:18,753 INFO [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732014858654 2024-11-19T11:14:18,764 DEBUG [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T11:14:18,765 INFO [RS_OPEN_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T11:14:18,766 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:14:18,769 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16c88966c0af,35833,1732014855640, state=OPEN 2024-11-19T11:14:18,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10152cd49a00001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T11:14:18,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T11:14:18,811 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T11:14:18,811 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T11:14:18,811 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:18,819 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-19T11:14:18,820 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16c88966c0af,35833,1732014855640 in 327 msec 2024-11-19T11:14:18,829 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-19T11:14:18,829 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 654 msec 2024-11-19T11:14:18,830 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T11:14:18,830 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-19T11:14:18,850 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:14:18,851 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:14:18,869 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:14:18,871 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48645, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:14:18,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6850 sec 2024-11-19T11:14:18,891 INFO [master/16c88966c0af:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732014858891, completionTime=-1 2024-11-19T11:14:18,894 INFO [master/16c88966c0af:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-19T11:14:18,894 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T11:14:18,919 INFO [master/16c88966c0af:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-19T11:14:18,919 INFO [master/16c88966c0af:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732014918919 2024-11-19T11:14:18,920 INFO [master/16c88966c0af:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732014978920 2024-11-19T11:14:18,920 INFO [master/16c88966c0af:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 25 msec 2024-11-19T11:14:18,922 INFO [master/16c88966c0af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16c88966c0af,41441,1732014854424-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T11:14:18,922 INFO [master/16c88966c0af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16c88966c0af,41441,1732014854424-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T11:14:18,922 INFO [master/16c88966c0af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16c88966c0af,41441,1732014854424-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T11:14:18,924 INFO [master/16c88966c0af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-16c88966c0af:41441, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T11:14:18,925 INFO [master/16c88966c0af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-19T11:14:18,925 INFO [master/16c88966c0af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-19T11:14:18,931 DEBUG [master/16c88966c0af:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-19T11:14:18,974 INFO [master/16c88966c0af:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.084sec 2024-11-19T11:14:18,975 INFO [master/16c88966c0af:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-19T11:14:18,976 INFO [master/16c88966c0af:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-19T11:14:18,977 INFO [master/16c88966c0af:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-19T11:14:18,981 INFO [master/16c88966c0af:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-19T11:14:18,981 INFO [master/16c88966c0af:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-19T11:14:18,982 INFO [master/16c88966c0af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16c88966c0af,41441,1732014854424-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T11:14:18,982 INFO [master/16c88966c0af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16c88966c0af,41441,1732014854424-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-19T11:14:18,991 DEBUG [master/16c88966c0af:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-19T11:14:18,992 INFO [master/16c88966c0af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-19T11:14:18,993 INFO [master/16c88966c0af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16c88966c0af,41441,1732014854424-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T11:14:19,078 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f081a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:19,080 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-19T11:14:19,081 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-19T11:14:19,083 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:14:19,085 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:14:19,092 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:14:19,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:14:19,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:14:19,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7607adcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:19,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:14:19,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:14:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:19,100 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:14:19,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62702105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:19,103 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:14:19,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:14:19,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:14:19,115 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:14:19,134 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=16c88966c0af,41441,1732014854424 2024-11-19T11:14:19,147 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=272, ProcessCount=11, AvailableMemoryMB=12466 2024-11-19T11:14:19,156 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T11:14:19,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 16c88966c0af,41441,1732014854424 2024-11-19T11:14:19,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@54ffeb4e 2024-11-19T11:14:19,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T11:14:19,169 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T11:14:19,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T11:14:19,177 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:14:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T11:14:19,190 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T11:14:19,193 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-19T11:14:19,193 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:19,196 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T11:14:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T11:14:19,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741835_1011 (size=1061) 2024-11-19T11:14:19,212 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094 2024-11-19T11:14:19,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741836_1012 (size=53) 2024-11-19T11:14:19,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T11:14:19,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T11:14:19,623 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:14:19,623 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9cbe0c20892be91e1583baaaf6ed665f, disabling compactions & flushes 2024-11-19T11:14:19,623 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:19,623 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:19,623 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. after waiting 0 ms 2024-11-19T11:14:19,623 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:19,623 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:19,624 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9cbe0c20892be91e1583baaaf6ed665f: Waiting for close lock at 1732014859623Disabling compacts and flushes for region at 1732014859623Disabling writes for close at 1732014859623Writing region close event to WAL at 1732014859623Closed at 1732014859623 2024-11-19T11:14:19,628 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T11:14:19,636 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732014859628"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732014859628"}]},"ts":"1732014859628"} 2024-11-19T11:14:19,640 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T11:14:19,642 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T11:14:19,645 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014859643"}]},"ts":"1732014859643"} 2024-11-19T11:14:19,649 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T11:14:19,650 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbe0c20892be91e1583baaaf6ed665f, ASSIGN}] 2024-11-19T11:14:19,653 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbe0c20892be91e1583baaaf6ed665f, ASSIGN 2024-11-19T11:14:19,655 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbe0c20892be91e1583baaaf6ed665f, ASSIGN; state=OFFLINE, location=16c88966c0af,35833,1732014855640; forceNewPlan=false, retain=false 2024-11-19T11:14:19,808 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9cbe0c20892be91e1583baaaf6ed665f, regionState=OPENING, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:14:19,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbe0c20892be91e1583baaaf6ed665f, ASSIGN because future has completed 2024-11-19T11:14:19,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640}] 2024-11-19T11:14:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T11:14:19,987 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:19,988 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} 2024-11-19T11:14:19,988 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:19,988 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:14:19,988 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:19,988 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:19,991 INFO [StoreOpener-9cbe0c20892be91e1583baaaf6ed665f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:19,993 INFO [StoreOpener-9cbe0c20892be91e1583baaaf6ed665f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:14:19,994 INFO [StoreOpener-9cbe0c20892be91e1583baaaf6ed665f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cbe0c20892be91e1583baaaf6ed665f columnFamilyName A 2024-11-19T11:14:19,994 DEBUG [StoreOpener-9cbe0c20892be91e1583baaaf6ed665f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:19,995 INFO [StoreOpener-9cbe0c20892be91e1583baaaf6ed665f-1 {}] regionserver.HStore(327): Store=9cbe0c20892be91e1583baaaf6ed665f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:14:19,995 INFO [StoreOpener-9cbe0c20892be91e1583baaaf6ed665f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:19,997 INFO [StoreOpener-9cbe0c20892be91e1583baaaf6ed665f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:14:19,998 INFO [StoreOpener-9cbe0c20892be91e1583baaaf6ed665f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cbe0c20892be91e1583baaaf6ed665f columnFamilyName B 2024-11-19T11:14:19,998 DEBUG [StoreOpener-9cbe0c20892be91e1583baaaf6ed665f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:19,999 INFO [StoreOpener-9cbe0c20892be91e1583baaaf6ed665f-1 {}] regionserver.HStore(327): Store=9cbe0c20892be91e1583baaaf6ed665f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:14:19,999 INFO [StoreOpener-9cbe0c20892be91e1583baaaf6ed665f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:20,000 INFO [StoreOpener-9cbe0c20892be91e1583baaaf6ed665f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:14:20,001 INFO [StoreOpener-9cbe0c20892be91e1583baaaf6ed665f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cbe0c20892be91e1583baaaf6ed665f columnFamilyName C 2024-11-19T11:14:20,001 DEBUG [StoreOpener-9cbe0c20892be91e1583baaaf6ed665f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:20,002 INFO [StoreOpener-9cbe0c20892be91e1583baaaf6ed665f-1 {}] regionserver.HStore(327): Store=9cbe0c20892be91e1583baaaf6ed665f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:14:20,002 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:20,003 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:20,004 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:20,005 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:20,006 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:20,007 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:20,007 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T11:14:20,010 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:20,013 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T11:14:20,014 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 9cbe0c20892be91e1583baaaf6ed665f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70574208, jitterRate=0.05163764953613281}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T11:14:20,014 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:20,015 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 9cbe0c20892be91e1583baaaf6ed665f: Running coprocessor pre-open hook at 1732014859989Writing region info on filesystem at 1732014859989Initializing all the Stores at 1732014859990 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014859990Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014859990Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014859990Cleaning up temporary data from old regions at 1732014860007 (+17 ms)Running coprocessor post-open hooks at 1732014860014 (+7 ms)Region opened successfully at 1732014860015 (+1 ms) 2024-11-19T11:14:20,018 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., pid=6, masterSystemTime=1732014859975 2024-11-19T11:14:20,022 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:20,022 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:20,023 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9cbe0c20892be91e1583baaaf6ed665f, regionState=OPEN, openSeqNum=2, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:14:20,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 because future has completed 2024-11-19T11:14:20,034 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-19T11:14:20,034 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 in 210 msec 2024-11-19T11:14:20,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-19T11:14:20,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbe0c20892be91e1583baaaf6ed665f, ASSIGN in 384 msec 2024-11-19T11:14:20,040 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T11:14:20,041 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014860040"}]},"ts":"1732014860040"} 2024-11-19T11:14:20,044 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T11:14:20,046 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T11:14:20,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 865 msec 2024-11-19T11:14:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T11:14:20,339 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:14:20,345 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bafc0a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,345 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:14:20,345 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:14:20,347 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:14:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:14:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:14:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46cd3e42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:14:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:14:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:20,351 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:14:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c82fed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,353 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:14:20,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:14:20,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:14:20,358 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:14:20,361 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T11:14:20,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 16c88966c0af,41441,1732014854424 2024-11-19T11:14:20,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@da7d497 2024-11-19T11:14:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T11:14:20,370 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T11:14:20,376 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@706bb323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,376 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:14:20,376 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:14:20,378 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:14:20,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:14:20,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:14:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@483a22c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:14:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:14:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:20,382 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:14:20,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,384 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:14:20,385 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:14:20,389 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:14:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:14:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:14:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cf4e74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:14:20,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:14:20,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:20,394 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:14:20,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf9665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,396 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:14:20,396 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:14:20,398 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:14:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:14:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:14:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:14:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:14:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:20,402 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:14:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27108bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,405 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:14:20,405 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:14:20,407 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:14:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:14:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:14:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0e6697, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:14:20,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:14:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:20,412 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:14:20,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7ca9c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,417 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:14:20,417 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:14:20,418 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:14:20,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:14:20,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:14:20,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2605ee65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:14:20,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:14:20,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:20,422 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:14:20,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f5127c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b274eea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,429 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:14:20,429 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:14:20,430 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:14:20,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:14:20,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:14:20,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a9c3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:14:20,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:14:20,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:20,434 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:14:20,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5346dc42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,436 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:14:20,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:14:20,438 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:14:20,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:14:20,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:14:20,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d111af0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:14:20,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:14:20,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:20,440 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:14:20,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e9639cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,442 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:14:20,443 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:14:20,444 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:14:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:14:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:14:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e406351, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:14:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:14:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:20,446 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:14:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d22a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,448 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:14:20,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:14:20,450 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:14:20,450 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:14:20,450 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:14:20,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c994daa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:14:20,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:14:20,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:20,452 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:14:20,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2171b18c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:20,458 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:14:20,459 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:14:20,459 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:14:20,459 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:14:20,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:14:20,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:14:20,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:14:20,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:14:20,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:14:20,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:14:20,462 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:14:20,463 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:14:20,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:14:20,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:14:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:14:20,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:14:20,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:14:20,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:14:20,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:14:20,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:14:20,467 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:14:20,467 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:14:20,468 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:14:20,467 DEBUG [Thread-160 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:14:20,468 DEBUG [Thread-160 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:14:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:14:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:14:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:14:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:14:20,470 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:14:20,471 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:14:20,471 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:14:20,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:14:20,475 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:14:20,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:14:20,477 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:14:20,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:14:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:14:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:14:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:14:20,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:14:20,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:14:20,484 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:14:20,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:14:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:14:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-19T11:14:20,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:14:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T11:14:20,496 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:14:20,498 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:14:20,500 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:14:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:20,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T11:14:20,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:20,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:20,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:20,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:20,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:20,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T11:14:20,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:20,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:20,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1732014920669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:20,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46270 deadline: 1732014920662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:20,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T11:14:20,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:20,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46208 deadline: 1732014920671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:20,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:20,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014920671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:20,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:20,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1732014920673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:20,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:20,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:20,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:20,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:20,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:20,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:20,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:20,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:20,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:20,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:20,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:20,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:20,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:20,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:20,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:20,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:20,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/2d652fdbd9ed44e8bc55b69324403caf is 50, key is test_row_0/A:col10/1732014860554/Put/seqid=0 2024-11-19T11:14:20,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:20,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741837_1013 (size=12001) 2024-11-19T11:14:20,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/2d652fdbd9ed44e8bc55b69324403caf 2024-11-19T11:14:20,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T11:14:20,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46270 deadline: 1732014920819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:20,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:20,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014920819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:20,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:20,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46208 deadline: 1732014920819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:20,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:20,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1732014920821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:20,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:20,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:20,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:20,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:20,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1732014920821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:20,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:20,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:20,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:20,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T11:14:20,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/f14c7178ac2848e4a713a5475bf5b401 is 50, key is test_row_0/B:col10/1732014860554/Put/seqid=0 2024-11-19T11:14:20,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:20,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:20,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:20,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:20,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:20,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:20,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741838_1014 (size=12001) 2024-11-19T11:14:20,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/f14c7178ac2848e4a713a5475bf5b401 2024-11-19T11:14:20,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/14ccb44a34344ebaa8bd49cae2a9cd38 is 50, key is test_row_0/C:col10/1732014860554/Put/seqid=0 2024-11-19T11:14:21,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741839_1015 (size=12001) 2024-11-19T11:14:21,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/14ccb44a34344ebaa8bd49cae2a9cd38 2024-11-19T11:14:21,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46270 deadline: 1732014921028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014921029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/2d652fdbd9ed44e8bc55b69324403caf as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/2d652fdbd9ed44e8bc55b69324403caf 2024-11-19T11:14:21,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1732014921040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46208 deadline: 1732014921039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1732014921040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T11:14:21,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/2d652fdbd9ed44e8bc55b69324403caf, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T11:14:21,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:21,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:21,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:21,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:21,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:21,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/f14c7178ac2848e4a713a5475bf5b401 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/f14c7178ac2848e4a713a5475bf5b401 2024-11-19T11:14:21,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/f14c7178ac2848e4a713a5475bf5b401, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T11:14:21,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/14ccb44a34344ebaa8bd49cae2a9cd38 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/14ccb44a34344ebaa8bd49cae2a9cd38 2024-11-19T11:14:21,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/14ccb44a34344ebaa8bd49cae2a9cd38, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T11:14:21,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 9cbe0c20892be91e1583baaaf6ed665f in 553ms, sequenceid=14, compaction requested=false 2024-11-19T11:14:21,111 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T11:14:21,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T11:14:21,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T11:14:21,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:21,218 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T11:14:21,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:21,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:21,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:21,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:21,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:21,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:21,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/682193c053594e4583687cf17401f0b0 is 50, key is test_row_0/A:col10/1732014860665/Put/seqid=0 2024-11-19T11:14:21,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741840_1016 (size=12001) 2024-11-19T11:14:21,253 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/682193c053594e4583687cf17401f0b0 2024-11-19T11:14:21,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/21ef635f5174481abf2cb45bf93e409c is 50, key is test_row_0/B:col10/1732014860665/Put/seqid=0 2024-11-19T11:14:21,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741841_1017 (size=12001) 2024-11-19T11:14:21,317 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/21ef635f5174481abf2cb45bf93e409c 2024-11-19T11:14:21,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/a41843ff19234afdb2d602a34857f26a is 50, key is test_row_0/C:col10/1732014860665/Put/seqid=0 2024-11-19T11:14:21,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:21,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:21,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741842_1018 (size=12001) 2024-11-19T11:14:21,357 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/a41843ff19234afdb2d602a34857f26a 2024-11-19T11:14:21,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/682193c053594e4583687cf17401f0b0 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/682193c053594e4583687cf17401f0b0 2024-11-19T11:14:21,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1732014921373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014921380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46270 deadline: 1732014921375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46208 deadline: 1732014921385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,399 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/682193c053594e4583687cf17401f0b0, entries=150, sequenceid=34, filesize=11.7 K 2024-11-19T11:14:21,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/21ef635f5174481abf2cb45bf93e409c as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/21ef635f5174481abf2cb45bf93e409c 2024-11-19T11:14:21,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1732014921392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,419 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/21ef635f5174481abf2cb45bf93e409c, entries=150, sequenceid=34, filesize=11.7 K 2024-11-19T11:14:21,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/a41843ff19234afdb2d602a34857f26a as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/a41843ff19234afdb2d602a34857f26a 2024-11-19T11:14:21,444 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/a41843ff19234afdb2d602a34857f26a, entries=150, sequenceid=34, filesize=11.7 K 2024-11-19T11:14:21,446 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9cbe0c20892be91e1583baaaf6ed665f in 228ms, sequenceid=34, compaction requested=false 2024-11-19T11:14:21,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:21,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:21,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-19T11:14:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-19T11:14:21,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-19T11:14:21,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 949 msec 2024-11-19T11:14:21,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 976 msec 2024-11-19T11:14:21,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:21,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T11:14:21,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:21,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:21,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:21,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:21,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:21,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:21,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/d5970b0d670a499086188ea7611e032f is 50, key is test_row_0/A:col10/1732014861504/Put/seqid=0 2024-11-19T11:14:21,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741843_1019 (size=12001) 2024-11-19T11:14:21,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/d5970b0d670a499086188ea7611e032f 2024-11-19T11:14:21,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46208 deadline: 1732014921554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1732014921558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/9b34eb4d18694fbe935fad1c5f148aae is 50, key is test_row_0/B:col10/1732014861504/Put/seqid=0 2024-11-19T11:14:21,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1732014921563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46270 deadline: 1732014921564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014921567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741844_1020 (size=12001) 2024-11-19T11:14:21,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/9b34eb4d18694fbe935fad1c5f148aae 2024-11-19T11:14:21,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/17f7ac7080964e5c8fbd4afad44d4676 is 50, key is test_row_0/C:col10/1732014861504/Put/seqid=0 2024-11-19T11:14:21,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T11:14:21,628 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:14:21,629 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:14:21,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-19T11:14:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T11:14:21,634 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:14:21,636 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:14:21,636 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:14:21,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741845_1021 (size=12001) 2024-11-19T11:14:21,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/17f7ac7080964e5c8fbd4afad44d4676 2024-11-19T11:14:21,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/d5970b0d670a499086188ea7611e032f as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/d5970b0d670a499086188ea7611e032f 2024-11-19T11:14:21,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46208 deadline: 1732014921678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1732014921678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/d5970b0d670a499086188ea7611e032f, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T11:14:21,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/9b34eb4d18694fbe935fad1c5f148aae as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/9b34eb4d18694fbe935fad1c5f148aae 2024-11-19T11:14:21,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1732014921688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014921689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46270 deadline: 1732014921690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/9b34eb4d18694fbe935fad1c5f148aae, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T11:14:21,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/17f7ac7080964e5c8fbd4afad44d4676 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/17f7ac7080964e5c8fbd4afad44d4676 2024-11-19T11:14:21,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/17f7ac7080964e5c8fbd4afad44d4676, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T11:14:21,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 9cbe0c20892be91e1583baaaf6ed665f in 227ms, sequenceid=48, compaction requested=true 2024-11-19T11:14:21,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:21,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:14:21,737 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T11:14:21,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:21,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:14:21,741 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:21,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:21,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:14:21,741 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:21,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:21,743 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/A is initiating minor compaction (all files) 2024-11-19T11:14:21,744 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:21,744 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/A in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:21,744 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/B is initiating minor compaction (all files) 2024-11-19T11:14:21,744 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/B in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:21,744 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/2d652fdbd9ed44e8bc55b69324403caf, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/682193c053594e4583687cf17401f0b0, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/d5970b0d670a499086188ea7611e032f] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=35.2 K 2024-11-19T11:14:21,744 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/f14c7178ac2848e4a713a5475bf5b401, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/21ef635f5174481abf2cb45bf93e409c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/9b34eb4d18694fbe935fad1c5f148aae] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=35.2 K 2024-11-19T11:14:21,746 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting f14c7178ac2848e4a713a5475bf5b401, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732014860550 2024-11-19T11:14:21,747 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 21ef635f5174481abf2cb45bf93e409c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732014860662 2024-11-19T11:14:21,748 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b34eb4d18694fbe935fad1c5f148aae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732014861373 2024-11-19T11:14:21,751 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d652fdbd9ed44e8bc55b69324403caf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732014860550 2024-11-19T11:14:21,752 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 682193c053594e4583687cf17401f0b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732014860662 2024-11-19T11:14:21,756 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5970b0d670a499086188ea7611e032f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732014861373 2024-11-19T11:14:21,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T11:14:21,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:21,792 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T11:14:21,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:21,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:21,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:21,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:21,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:21,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:21,795 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#A#compaction#9 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:21,796 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#B#compaction#10 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:21,796 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/d2e69394289f44b08dcc6e753b3db538 is 50, key is test_row_0/A:col10/1732014861504/Put/seqid=0 2024-11-19T11:14:21,797 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/2ae96b7af8fa44ce9a6a6db06f31c4a3 is 50, key is test_row_0/B:col10/1732014861504/Put/seqid=0 2024-11-19T11:14:21,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/7d3351937a794cd3b66765557b56a24f is 50, key is test_row_0/A:col10/1732014861550/Put/seqid=0 2024-11-19T11:14:21,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741846_1022 (size=12104) 2024-11-19T11:14:21,853 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/d2e69394289f44b08dcc6e753b3db538 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/d2e69394289f44b08dcc6e753b3db538 2024-11-19T11:14:21,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741847_1023 (size=12104) 2024-11-19T11:14:21,872 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/2ae96b7af8fa44ce9a6a6db06f31c4a3 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/2ae96b7af8fa44ce9a6a6db06f31c4a3 2024-11-19T11:14:21,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741848_1024 (size=12001) 2024-11-19T11:14:21,897 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/B of 9cbe0c20892be91e1583baaaf6ed665f into 2ae96b7af8fa44ce9a6a6db06f31c4a3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:21,897 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:21,900 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/B, priority=13, startTime=1732014861741; duration=0sec 2024-11-19T11:14:21,901 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:21,901 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:B 2024-11-19T11:14:21,901 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:21,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:21,904 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/A of 9cbe0c20892be91e1583baaaf6ed665f into d2e69394289f44b08dcc6e753b3db538(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:21,904 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:21,904 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/A, priority=13, startTime=1732014861733; duration=0sec 2024-11-19T11:14:21,904 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:21,904 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:A 2024-11-19T11:14:21,907 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:21,907 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/C is initiating minor compaction (all files) 2024-11-19T11:14:21,907 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/C in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:21,907 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/14ccb44a34344ebaa8bd49cae2a9cd38, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/a41843ff19234afdb2d602a34857f26a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/17f7ac7080964e5c8fbd4afad44d4676] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=35.2 K 2024-11-19T11:14:21,908 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 14ccb44a34344ebaa8bd49cae2a9cd38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732014860550 2024-11-19T11:14:21,910 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting a41843ff19234afdb2d602a34857f26a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732014860662 2024-11-19T11:14:21,912 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 17f7ac7080964e5c8fbd4afad44d4676, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732014861373 2024-11-19T11:14:21,935 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#C#compaction#12 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:21,936 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/3a771587a42c4df3a13b23e96a6a3fdc is 50, key is test_row_0/C:col10/1732014861504/Put/seqid=0 2024-11-19T11:14:21,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1732014921935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T11:14:21,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1732014921939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46270 deadline: 1732014921939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46208 deadline: 1732014921943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014921943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:21,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:21,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:21,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741849_1025 (size=12104) 2024-11-19T11:14:21,982 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/3a771587a42c4df3a13b23e96a6a3fdc as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/3a771587a42c4df3a13b23e96a6a3fdc 2024-11-19T11:14:22,002 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/C of 9cbe0c20892be91e1583baaaf6ed665f into 3a771587a42c4df3a13b23e96a6a3fdc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:22,003 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:22,003 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/C, priority=13, startTime=1732014861741; duration=0sec 2024-11-19T11:14:22,003 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:22,003 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:C 2024-11-19T11:14:22,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1732014922048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1732014922069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46208 deadline: 1732014922069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46270 deadline: 1732014922069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014922070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T11:14:22,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1732014922259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1732014922278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,285 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/7d3351937a794cd3b66765557b56a24f 2024-11-19T11:14:22,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46270 deadline: 1732014922287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46208 deadline: 1732014922287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014922288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/b52a384872ca413f8680c81e8064656e is 50, key is test_row_0/B:col10/1732014861550/Put/seqid=0 2024-11-19T11:14:22,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741850_1026 (size=12001) 2024-11-19T11:14:22,328 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/b52a384872ca413f8680c81e8064656e 2024-11-19T11:14:22,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/ad95f1f97efd443b90d8a90d35b4aad8 is 50, key is test_row_0/C:col10/1732014861550/Put/seqid=0 2024-11-19T11:14:22,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741851_1027 (size=12001) 2024-11-19T11:14:22,383 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/ad95f1f97efd443b90d8a90d35b4aad8 2024-11-19T11:14:22,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/7d3351937a794cd3b66765557b56a24f as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/7d3351937a794cd3b66765557b56a24f 2024-11-19T11:14:22,420 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/7d3351937a794cd3b66765557b56a24f, entries=150, sequenceid=66, filesize=11.7 K 2024-11-19T11:14:22,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/b52a384872ca413f8680c81e8064656e as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/b52a384872ca413f8680c81e8064656e 2024-11-19T11:14:22,440 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/b52a384872ca413f8680c81e8064656e, entries=150, sequenceid=66, filesize=11.7 K 2024-11-19T11:14:22,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/ad95f1f97efd443b90d8a90d35b4aad8 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/ad95f1f97efd443b90d8a90d35b4aad8 2024-11-19T11:14:22,461 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/ad95f1f97efd443b90d8a90d35b4aad8, entries=150, sequenceid=66, filesize=11.7 K 2024-11-19T11:14:22,465 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 9cbe0c20892be91e1583baaaf6ed665f in 672ms, sequenceid=66, compaction requested=false 2024-11-19T11:14:22,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:22,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:22,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-19T11:14:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-19T11:14:22,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-19T11:14:22,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 834 msec 2024-11-19T11:14:22,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 848 msec 2024-11-19T11:14:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:22,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T11:14:22,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:22,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:22,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:22,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:22,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:22,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:22,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/340057bdf5ca43e6b2ac9d93205e19b7 is 50, key is test_row_0/A:col10/1732014861938/Put/seqid=0 2024-11-19T11:14:22,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741852_1028 (size=14341) 2024-11-19T11:14:22,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/340057bdf5ca43e6b2ac9d93205e19b7 2024-11-19T11:14:22,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1732014922626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1732014922635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46270 deadline: 1732014922637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46208 deadline: 1732014922638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014922641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/7a3bea00bf1b40198568390ca3675a56 is 50, key is test_row_0/B:col10/1732014861938/Put/seqid=0 2024-11-19T11:14:22,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741853_1029 (size=12001) 2024-11-19T11:14:22,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1732014922748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1732014922748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014922759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46270 deadline: 1732014922758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46208 deadline: 1732014922760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T11:14:22,769 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:14:22,769 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:14:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-19T11:14:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T11:14:22,775 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:14:22,777 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:14:22,778 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:14:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T11:14:22,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T11:14:22,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:22,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:22,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:22,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:22,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:22,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1732014922958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1732014922959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46208 deadline: 1732014922968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46270 deadline: 1732014922968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:22,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014922968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:22,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:22,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:23,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/7a3bea00bf1b40198568390ca3675a56 2024-11-19T11:14:23,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T11:14:23,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:23,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:23,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:23,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:23,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T11:14:23,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/7f3fbd4440234864b3d40dbac158c582 is 50, key is test_row_0/C:col10/1732014861938/Put/seqid=0 2024-11-19T11:14:23,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741854_1030 (size=12001) 2024-11-19T11:14:23,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T11:14:23,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:23,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:23,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:23,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:23,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:23,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1732014923268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:23,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1732014923268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:23,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:23,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:23,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46270 deadline: 1732014923287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:23,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:23,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46208 deadline: 1732014923287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:23,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:23,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014923298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:23,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:23,391 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-19T11:14:23,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T11:14:23,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:23,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:23,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:23,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:23,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T11:14:23,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/7f3fbd4440234864b3d40dbac158c582 2024-11-19T11:14:23,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/340057bdf5ca43e6b2ac9d93205e19b7 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/340057bdf5ca43e6b2ac9d93205e19b7 2024-11-19T11:14:23,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/340057bdf5ca43e6b2ac9d93205e19b7, entries=200, sequenceid=84, filesize=14.0 K 2024-11-19T11:14:23,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/7a3bea00bf1b40198568390ca3675a56 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/7a3bea00bf1b40198568390ca3675a56 2024-11-19T11:14:23,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/7a3bea00bf1b40198568390ca3675a56, entries=150, sequenceid=84, filesize=11.7 K 2024-11-19T11:14:23,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T11:14:23,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/7f3fbd4440234864b3d40dbac158c582 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/7f3fbd4440234864b3d40dbac158c582 2024-11-19T11:14:23,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:23,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:23,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:23,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:23,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:23,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/7f3fbd4440234864b3d40dbac158c582, entries=150, sequenceid=84, filesize=11.7 K 2024-11-19T11:14:23,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 9cbe0c20892be91e1583baaaf6ed665f in 1007ms, sequenceid=84, compaction requested=true 2024-11-19T11:14:23,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:23,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:14:23,582 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:23,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:23,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:14:23,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:23,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:14:23,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T11:14:23,582 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:23,587 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:23,587 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:23,588 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/A is initiating minor compaction (all files) 2024-11-19T11:14:23,588 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/B is initiating minor compaction (all files) 2024-11-19T11:14:23,588 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/A in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:23,588 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/B in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:23,588 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/d2e69394289f44b08dcc6e753b3db538, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/7d3351937a794cd3b66765557b56a24f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/340057bdf5ca43e6b2ac9d93205e19b7] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=37.5 K 2024-11-19T11:14:23,588 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/2ae96b7af8fa44ce9a6a6db06f31c4a3, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/b52a384872ca413f8680c81e8064656e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/7a3bea00bf1b40198568390ca3675a56] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=35.3 K 2024-11-19T11:14:23,589 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ae96b7af8fa44ce9a6a6db06f31c4a3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732014861373 2024-11-19T11:14:23,590 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2e69394289f44b08dcc6e753b3db538, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732014861373 2024-11-19T11:14:23,592 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d3351937a794cd3b66765557b56a24f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732014861550 2024-11-19T11:14:23,596 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting b52a384872ca413f8680c81e8064656e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732014861550 2024-11-19T11:14:23,597 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 340057bdf5ca43e6b2ac9d93205e19b7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732014861932 2024-11-19T11:14:23,598 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a3bea00bf1b40198568390ca3675a56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732014861932 2024-11-19T11:14:23,627 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:23,627 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:23,628 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/479f4110df1c450d89df820f6e747ded is 50, key is test_row_0/A:col10/1732014861938/Put/seqid=0 2024-11-19T11:14:23,630 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/92d182811b7c488497d49006de32bcd8 is 50, key is test_row_0/B:col10/1732014861938/Put/seqid=0 2024-11-19T11:14:23,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741855_1031 (size=12207) 2024-11-19T11:14:23,677 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/479f4110df1c450d89df820f6e747ded as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/479f4110df1c450d89df820f6e747ded 2024-11-19T11:14:23,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741856_1032 (size=12207) 2024-11-19T11:14:23,698 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/A of 9cbe0c20892be91e1583baaaf6ed665f into 479f4110df1c450d89df820f6e747ded(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:23,698 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:23,698 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/A, priority=13, startTime=1732014863582; duration=0sec 2024-11-19T11:14:23,698 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:23,698 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:A 2024-11-19T11:14:23,699 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:23,701 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:23,702 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/C is initiating minor compaction (all files) 2024-11-19T11:14:23,702 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/C in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:23,702 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/3a771587a42c4df3a13b23e96a6a3fdc, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/ad95f1f97efd443b90d8a90d35b4aad8, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/7f3fbd4440234864b3d40dbac158c582] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=35.3 K 2024-11-19T11:14:23,703 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a771587a42c4df3a13b23e96a6a3fdc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732014861373 2024-11-19T11:14:23,704 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad95f1f97efd443b90d8a90d35b4aad8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732014861550 2024-11-19T11:14:23,704 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f3fbd4440234864b3d40dbac158c582, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732014861932 2024-11-19T11:14:23,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T11:14:23,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:23,711 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T11:14:23,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:23,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:23,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:23,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:23,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:23,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:23,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/8086b01099c54b26bda3b74bcb8f5614 is 50, key is test_row_0/A:col10/1732014862622/Put/seqid=0 2024-11-19T11:14:23,741 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:23,742 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/5a1d40f3446044c0813679190886a33b is 50, key is test_row_0/C:col10/1732014861938/Put/seqid=0 2024-11-19T11:14:23,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741857_1033 (size=12001) 2024-11-19T11:14:23,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741858_1034 (size=12207) 2024-11-19T11:14:23,778 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/5a1d40f3446044c0813679190886a33b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/5a1d40f3446044c0813679190886a33b 2024-11-19T11:14:23,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:23,802 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/C of 9cbe0c20892be91e1583baaaf6ed665f into 5a1d40f3446044c0813679190886a33b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:23,802 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:23,802 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/C, priority=13, startTime=1732014863582; duration=0sec 2024-11-19T11:14:23,802 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:23,802 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:C 2024-11-19T11:14:23,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1732014923810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:23,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014923817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:23,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:23,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:23,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46270 deadline: 1732014923872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:23,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:23,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1732014923872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:23,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:23,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46208 deadline: 1732014923874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:23,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T11:14:23,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1732014923977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:23,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46270 deadline: 1732014923988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:23,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46208 deadline: 1732014923988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1732014923989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:24,104 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/92d182811b7c488497d49006de32bcd8 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/92d182811b7c488497d49006de32bcd8 2024-11-19T11:14:24,119 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/B of 9cbe0c20892be91e1583baaaf6ed665f into 92d182811b7c488497d49006de32bcd8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:24,119 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:24,119 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/B, priority=13, startTime=1732014863582; duration=0sec 2024-11-19T11:14:24,120 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:24,120 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:B 2024-11-19T11:14:24,146 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/8086b01099c54b26bda3b74bcb8f5614 2024-11-19T11:14:24,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/c20f7421282a41f6838a333dddaa429f is 50, key is test_row_0/B:col10/1732014862622/Put/seqid=0 2024-11-19T11:14:24,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46270 deadline: 1732014924197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:24,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:24,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:24,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:24,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:24,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:24,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46208 deadline: 1732014924198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:24,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1732014924197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:24,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:24,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:24,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:24,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:24,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:24,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:24,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:24,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1732014924198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:24,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:24,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:24,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:24,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741859_1035 (size=12001) 2024-11-19T11:14:24,214 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/c20f7421282a41f6838a333dddaa429f 2024-11-19T11:14:24,231 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T11:14:24,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/97d753cdef7844e8982c019d4f8d129f is 50, key is test_row_0/C:col10/1732014862622/Put/seqid=0 2024-11-19T11:14:24,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741860_1036 (size=12001) 2024-11-19T11:14:24,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:24,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46270 deadline: 1732014924507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1732014924508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:24,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:24,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:24,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:24,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46208 deadline: 1732014924508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:24,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:24,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:24,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:24,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:24,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:24,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:24,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1732014924517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:24,675 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/97d753cdef7844e8982c019d4f8d129f 2024-11-19T11:14:24,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/8086b01099c54b26bda3b74bcb8f5614 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/8086b01099c54b26bda3b74bcb8f5614 2024-11-19T11:14:24,698 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/8086b01099c54b26bda3b74bcb8f5614, entries=150, sequenceid=103, filesize=11.7 K 2024-11-19T11:14:24,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/c20f7421282a41f6838a333dddaa429f as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/c20f7421282a41f6838a333dddaa429f 2024-11-19T11:14:24,709 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/c20f7421282a41f6838a333dddaa429f, entries=150, sequenceid=103, filesize=11.7 K 2024-11-19T11:14:24,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/97d753cdef7844e8982c019d4f8d129f as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/97d753cdef7844e8982c019d4f8d129f 2024-11-19T11:14:24,724 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/97d753cdef7844e8982c019d4f8d129f, entries=150, sequenceid=103, filesize=11.7 K 2024-11-19T11:14:24,726 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 9cbe0c20892be91e1583baaaf6ed665f in 1016ms, sequenceid=103, compaction requested=false 2024-11-19T11:14:24,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:24,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:24,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-19T11:14:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-19T11:14:24,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-19T11:14:24,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9530 sec 2024-11-19T11:14:24,739 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.9660 sec 2024-11-19T11:14:24,759 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-19T11:14:24,760 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-19T11:14:24,762 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T11:14:24,763 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-19T11:14:24,763 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-19T11:14:24,763 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-19T11:14:24,764 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T11:14:24,764 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T11:14:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:24,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T11:14:24,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:24,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:24,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:24,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:24,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:24,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:24,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/fa710919571247ac91ecedecf8297d83 is 50, key is test_row_0/A:col10/1732014864888/Put/seqid=0 2024-11-19T11:14:24,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741861_1037 (size=14341) 2024-11-19T11:14:24,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/fa710919571247ac91ecedecf8297d83 2024-11-19T11:14:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T11:14:24,938 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:14:24,939 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:14:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-19T11:14:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T11:14:24,944 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:14:24,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/c57a216f5d874aa6a98bfc7342af4ac5 is 50, key is test_row_0/B:col10/1732014864888/Put/seqid=0 2024-11-19T11:14:24,947 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:14:24,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:14:24,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741862_1038 (size=12001) 2024-11-19T11:14:24,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014924974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:25,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46270 deadline: 1732014925018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:25,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1732014925019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:25,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:25,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46208 deadline: 1732014925019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:25,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:25,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:25,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:25,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1732014925028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T11:14:25,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014925088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:25,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T11:14:25,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:25,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:25,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:25,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:25,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:25,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:25,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T11:14:25,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:25,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:25,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:25,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:25,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:25,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:25,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T11:14:25,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:25,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014925297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:25,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:25,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:25,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:25,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/c57a216f5d874aa6a98bfc7342af4ac5 2024-11-19T11:14:25,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/7fdcafeddb4c49219f1102a1dadd996b is 50, key is test_row_0/C:col10/1732014864888/Put/seqid=0 2024-11-19T11:14:25,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741863_1039 (size=12001) 2024-11-19T11:14:25,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/7fdcafeddb4c49219f1102a1dadd996b 2024-11-19T11:14:25,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T11:14:25,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:25,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:25,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:25,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:25,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:25,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:25,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/fa710919571247ac91ecedecf8297d83 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/fa710919571247ac91ecedecf8297d83 2024-11-19T11:14:25,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/fa710919571247ac91ecedecf8297d83, entries=200, sequenceid=120, filesize=14.0 K 2024-11-19T11:14:25,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/c57a216f5d874aa6a98bfc7342af4ac5 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/c57a216f5d874aa6a98bfc7342af4ac5 2024-11-19T11:14:25,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/c57a216f5d874aa6a98bfc7342af4ac5, entries=150, sequenceid=120, filesize=11.7 K 2024-11-19T11:14:25,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/7fdcafeddb4c49219f1102a1dadd996b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/7fdcafeddb4c49219f1102a1dadd996b 2024-11-19T11:14:25,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/7fdcafeddb4c49219f1102a1dadd996b, entries=150, sequenceid=120, filesize=11.7 K 2024-11-19T11:14:25,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 9cbe0c20892be91e1583baaaf6ed665f in 559ms, sequenceid=120, compaction requested=true 2024-11-19T11:14:25,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:25,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:14:25,451 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:25,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:25,451 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:25,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:14:25,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:25,453 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:25,453 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:25,453 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/B is initiating minor compaction (all files) 2024-11-19T11:14:25,453 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/A is initiating minor compaction (all files) 2024-11-19T11:14:25,453 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/B in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:25,453 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/A in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:25,453 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/92d182811b7c488497d49006de32bcd8, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/c20f7421282a41f6838a333dddaa429f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/c57a216f5d874aa6a98bfc7342af4ac5] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=35.4 K 2024-11-19T11:14:25,453 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/479f4110df1c450d89df820f6e747ded, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/8086b01099c54b26bda3b74bcb8f5614, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/fa710919571247ac91ecedecf8297d83] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=37.6 K 2024-11-19T11:14:25,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:14:25,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:25,454 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 92d182811b7c488497d49006de32bcd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732014861932 2024-11-19T11:14:25,455 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 479f4110df1c450d89df820f6e747ded, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732014861932 2024-11-19T11:14:25,455 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting c20f7421282a41f6838a333dddaa429f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732014862609 2024-11-19T11:14:25,455 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8086b01099c54b26bda3b74bcb8f5614, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732014862609 2024-11-19T11:14:25,456 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa710919571247ac91ecedecf8297d83, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732014863786 2024-11-19T11:14:25,456 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting c57a216f5d874aa6a98bfc7342af4ac5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732014863807 2024-11-19T11:14:25,475 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#B#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:25,479 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/fe1bf774582149b7826df2b1eb6e45f3 is 50, key is test_row_0/B:col10/1732014864888/Put/seqid=0 2024-11-19T11:14:25,481 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#A#compaction#28 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:25,482 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/eb405d91f113406a8802392532074764 is 50, key is test_row_0/A:col10/1732014864888/Put/seqid=0 2024-11-19T11:14:25,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741864_1040 (size=12309) 2024-11-19T11:14:25,531 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/fe1bf774582149b7826df2b1eb6e45f3 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/fe1bf774582149b7826df2b1eb6e45f3 2024-11-19T11:14:25,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741865_1041 (size=12309) 2024-11-19T11:14:25,549 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/B of 9cbe0c20892be91e1583baaaf6ed665f into fe1bf774582149b7826df2b1eb6e45f3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:25,550 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:25,550 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/B, priority=13, startTime=1732014865451; duration=0sec 2024-11-19T11:14:25,550 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:25,550 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:B 2024-11-19T11:14:25,551 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:25,554 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:25,554 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/C is initiating minor compaction (all files) 2024-11-19T11:14:25,554 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/C in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:25,555 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/5a1d40f3446044c0813679190886a33b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/97d753cdef7844e8982c019d4f8d129f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/7fdcafeddb4c49219f1102a1dadd996b] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=35.4 K 2024-11-19T11:14:25,555 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a1d40f3446044c0813679190886a33b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732014861932 2024-11-19T11:14:25,556 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 97d753cdef7844e8982c019d4f8d129f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732014862609 2024-11-19T11:14:25,557 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fdcafeddb4c49219f1102a1dadd996b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732014863807 2024-11-19T11:14:25,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T11:14:25,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:25,564 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T11:14:25,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:25,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:25,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:25,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:25,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:25,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:25,567 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/eb405d91f113406a8802392532074764 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/eb405d91f113406a8802392532074764 2024-11-19T11:14:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T11:14:25,579 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/A of 9cbe0c20892be91e1583baaaf6ed665f into eb405d91f113406a8802392532074764(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:25,579 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:25,579 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/A, priority=13, startTime=1732014865451; duration=0sec 2024-11-19T11:14:25,580 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:25,580 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:A 2024-11-19T11:14:25,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/e16ec392e0d64a96adb162d328243069 is 50, key is test_row_0/A:col10/1732014864912/Put/seqid=0 2024-11-19T11:14:25,596 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#C#compaction#30 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:25,597 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/f2a0240926ca495bad73bdf7351f0f2d is 50, key is test_row_0/C:col10/1732014864888/Put/seqid=0 2024-11-19T11:14:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:25,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:25,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741866_1042 (size=12101) 2024-11-19T11:14:25,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741867_1043 (size=12309) 2024-11-19T11:14:25,671 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/f2a0240926ca495bad73bdf7351f0f2d as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/f2a0240926ca495bad73bdf7351f0f2d 2024-11-19T11:14:25,683 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/C of 9cbe0c20892be91e1583baaaf6ed665f into f2a0240926ca495bad73bdf7351f0f2d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:25,683 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:25,683 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/C, priority=13, startTime=1732014865451; duration=0sec 2024-11-19T11:14:25,683 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:25,684 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:C 2024-11-19T11:14:25,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014925742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:25,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014925857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:26,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46208 deadline: 1732014926028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:26,033 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/e16ec392e0d64a96adb162d328243069 2024-11-19T11:14:26,037 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_2' in 9cbe0c20892be91e1583baaaf6ed665f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:14:26,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:26,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:26,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:26,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46270 deadline: 1732014926038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:26,039 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_1' in 9cbe0c20892be91e1583baaaf6ed665f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:14:26,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:26,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:26,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:26,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1732014926039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:26,045 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_2' in 9cbe0c20892be91e1583baaaf6ed665f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:14:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:26,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/f5a743c5865c49edb8941510284d198c is 50, key is test_row_0/B:col10/1732014864912/Put/seqid=0 2024-11-19T11:14:26,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1732014926048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:26,051 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_1' in 9cbe0c20892be91e1583baaaf6ed665f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:14:26,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:26,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:26,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:26,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741868_1044 (size=12101) 2024-11-19T11:14:26,062 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/f5a743c5865c49edb8941510284d198c 2024-11-19T11:14:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T11:14:26,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/99da47e8efbf4695a5bc6b414c70e3bd is 50, key is test_row_0/C:col10/1732014864912/Put/seqid=0 2024-11-19T11:14:26,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014926078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:26,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:26,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:26,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:26,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741869_1045 (size=12101) 2024-11-19T11:14:26,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014926388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:26,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:26,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:26,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:26,505 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/99da47e8efbf4695a5bc6b414c70e3bd 2024-11-19T11:14:26,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/e16ec392e0d64a96adb162d328243069 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/e16ec392e0d64a96adb162d328243069 2024-11-19T11:14:26,519 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/e16ec392e0d64a96adb162d328243069, entries=150, sequenceid=138, filesize=11.8 K 2024-11-19T11:14:26,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/f5a743c5865c49edb8941510284d198c as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/f5a743c5865c49edb8941510284d198c 2024-11-19T11:14:26,528 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/f5a743c5865c49edb8941510284d198c, entries=150, sequenceid=138, filesize=11.8 K 2024-11-19T11:14:26,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/99da47e8efbf4695a5bc6b414c70e3bd as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/99da47e8efbf4695a5bc6b414c70e3bd 2024-11-19T11:14:26,539 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/99da47e8efbf4695a5bc6b414c70e3bd, entries=150, sequenceid=138, filesize=11.8 K 2024-11-19T11:14:26,541 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 9cbe0c20892be91e1583baaaf6ed665f in 977ms, sequenceid=138, compaction requested=false 2024-11-19T11:14:26,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:26,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:26,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-19T11:14:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-19T11:14:26,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-19T11:14:26,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5960 sec 2024-11-19T11:14:26,549 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.6080 sec 2024-11-19T11:14:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:26,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T11:14:26,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:26,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:26,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:26,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:26,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:26,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:26,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/c239eb61378645f0a3219059d57c3ce1 is 50, key is test_row_0/A:col10/1732014865706/Put/seqid=0 2024-11-19T11:14:26,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741870_1046 (size=14541) 2024-11-19T11:14:26,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/c239eb61378645f0a3219059d57c3ce1 2024-11-19T11:14:26,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/89b5e010007d4cc5beec1212eb610097 is 50, key is test_row_0/B:col10/1732014865706/Put/seqid=0 2024-11-19T11:14:26,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741871_1047 (size=12151) 2024-11-19T11:14:26,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/89b5e010007d4cc5beec1212eb610097 2024-11-19T11:14:26,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014926973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:26,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:26,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:26,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:26,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/0409ac796794492cab1975efee553d6e is 50, key is test_row_0/C:col10/1732014865706/Put/seqid=0 2024-11-19T11:14:27,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741872_1048 (size=12151) 2024-11-19T11:14:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T11:14:27,088 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:14:27,088 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:14:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-19T11:14:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T11:14:27,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014927088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:27,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:27,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:27,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:27,092 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:14:27,093 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:14:27,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:14:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T11:14:27,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T11:14:27,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:27,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:27,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:27,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:27,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:27,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014927298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:27,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T11:14:27,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:27,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:27,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:27,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:27,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T11:14:27,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/0409ac796794492cab1975efee553d6e 2024-11-19T11:14:27,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/c239eb61378645f0a3219059d57c3ce1 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/c239eb61378645f0a3219059d57c3ce1 2024-11-19T11:14:27,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/c239eb61378645f0a3219059d57c3ce1, entries=200, sequenceid=156, filesize=14.2 K 2024-11-19T11:14:27,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/89b5e010007d4cc5beec1212eb610097 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/89b5e010007d4cc5beec1212eb610097 2024-11-19T11:14:27,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/89b5e010007d4cc5beec1212eb610097, entries=150, sequenceid=156, filesize=11.9 K 2024-11-19T11:14:27,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/0409ac796794492cab1975efee553d6e as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/0409ac796794492cab1975efee553d6e 2024-11-19T11:14:27,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/0409ac796794492cab1975efee553d6e, entries=150, sequenceid=156, filesize=11.9 K 2024-11-19T11:14:27,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 9cbe0c20892be91e1583baaaf6ed665f in 569ms, sequenceid=156, compaction requested=true 2024-11-19T11:14:27,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:27,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:14:27,469 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:27,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:27,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:14:27,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:27,469 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:27,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:14:27,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:27,470 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36561 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:27,470 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38951 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:27,470 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/A is initiating minor compaction (all files) 2024-11-19T11:14:27,470 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/B is initiating minor compaction (all files) 2024-11-19T11:14:27,470 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/A in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:27,470 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/B in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:27,470 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/fe1bf774582149b7826df2b1eb6e45f3, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/f5a743c5865c49edb8941510284d198c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/89b5e010007d4cc5beec1212eb610097] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=35.7 K 2024-11-19T11:14:27,470 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/eb405d91f113406a8802392532074764, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/e16ec392e0d64a96adb162d328243069, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/c239eb61378645f0a3219059d57c3ce1] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=38.0 K 2024-11-19T11:14:27,471 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting fe1bf774582149b7826df2b1eb6e45f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732014863807 2024-11-19T11:14:27,471 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb405d91f113406a8802392532074764, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732014863807 2024-11-19T11:14:27,471 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting f5a743c5865c49edb8941510284d198c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732014864912 2024-11-19T11:14:27,471 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting e16ec392e0d64a96adb162d328243069, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732014864912 2024-11-19T11:14:27,471 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 89b5e010007d4cc5beec1212eb610097, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732014865706 2024-11-19T11:14:27,473 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting c239eb61378645f0a3219059d57c3ce1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732014865706 2024-11-19T11:14:27,500 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#A#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:27,501 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/6400a5d6fd5d4209911c99c56a691152 is 50, key is test_row_0/A:col10/1732014865706/Put/seqid=0 2024-11-19T11:14:27,503 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#B#compaction#37 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:27,503 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/7fe5a07da1e9437d9d2d8fb50814590c is 50, key is test_row_0/B:col10/1732014865706/Put/seqid=0 2024-11-19T11:14:27,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741873_1049 (size=12561) 2024-11-19T11:14:27,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741874_1050 (size=12561) 2024-11-19T11:14:27,547 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/6400a5d6fd5d4209911c99c56a691152 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/6400a5d6fd5d4209911c99c56a691152 2024-11-19T11:14:27,555 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/A of 9cbe0c20892be91e1583baaaf6ed665f into 6400a5d6fd5d4209911c99c56a691152(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:27,555 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:27,555 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/A, priority=13, startTime=1732014867468; duration=0sec 2024-11-19T11:14:27,555 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:27,555 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:A 2024-11-19T11:14:27,555 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:27,557 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36561 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:27,557 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/C is initiating minor compaction (all files) 2024-11-19T11:14:27,557 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/C in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:27,557 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/f2a0240926ca495bad73bdf7351f0f2d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/99da47e8efbf4695a5bc6b414c70e3bd, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/0409ac796794492cab1975efee553d6e] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=35.7 K 2024-11-19T11:14:27,558 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2a0240926ca495bad73bdf7351f0f2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732014863807 2024-11-19T11:14:27,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T11:14:27,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:27,560 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T11:14:27,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:27,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:27,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:27,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:27,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:27,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:27,561 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99da47e8efbf4695a5bc6b414c70e3bd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732014864912 2024-11-19T11:14:27,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/c04c2fd9254546e1b8f9c71f7d871594 is 50, key is test_row_0/A:col10/1732014866963/Put/seqid=0 2024-11-19T11:14:27,572 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0409ac796794492cab1975efee553d6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732014865706 2024-11-19T11:14:27,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741875_1051 (size=12151) 2024-11-19T11:14:27,600 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#C#compaction#39 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:27,601 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/23b039d7678741889d9fa12ce6f6480c is 50, key is test_row_0/C:col10/1732014865706/Put/seqid=0 2024-11-19T11:14:27,601 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/c04c2fd9254546e1b8f9c71f7d871594 2024-11-19T11:14:27,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/b03b4a3a415442f2a7bd95208a52e831 is 50, key is test_row_0/B:col10/1732014866963/Put/seqid=0 2024-11-19T11:14:27,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741877_1053 (size=12151) 2024-11-19T11:14:27,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741876_1052 (size=12561) 2024-11-19T11:14:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:27,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T11:14:27,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014927721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:27,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:27,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:27,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:27,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014927837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:27,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:27,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:27,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:27,956 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/7fe5a07da1e9437d9d2d8fb50814590c as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/7fe5a07da1e9437d9d2d8fb50814590c 2024-11-19T11:14:27,970 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/B of 9cbe0c20892be91e1583baaaf6ed665f into 7fe5a07da1e9437d9d2d8fb50814590c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:27,971 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:27,971 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/B, priority=13, startTime=1732014867469; duration=0sec 2024-11-19T11:14:27,971 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:27,971 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:B 2024-11-19T11:14:28,022 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/b03b4a3a415442f2a7bd95208a52e831 2024-11-19T11:14:28,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014928048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:28,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:28,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:28,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:28,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/427d5dc3046b4adfaab4c17a9c767802 is 50, key is test_row_0/C:col10/1732014866963/Put/seqid=0 2024-11-19T11:14:28,061 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/23b039d7678741889d9fa12ce6f6480c as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/23b039d7678741889d9fa12ce6f6480c 2024-11-19T11:14:28,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:28,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46270 deadline: 1732014928059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46208 deadline: 1732014928059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:28,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1732014928059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:28,063 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_2' in 9cbe0c20892be91e1583baaaf6ed665f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:14:28,063 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_1' in 9cbe0c20892be91e1583baaaf6ed665f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:14:28,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:28,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:28,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:28,064 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_2' in 9cbe0c20892be91e1583baaaf6ed665f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:14:28,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:28,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:28,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:28,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:28,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:28,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:28,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1732014928068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:28,073 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_1' in 9cbe0c20892be91e1583baaaf6ed665f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:14:28,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:28,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:28,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:28,077 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/C of 9cbe0c20892be91e1583baaaf6ed665f into 23b039d7678741889d9fa12ce6f6480c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:28,078 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:28,078 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/C, priority=13, startTime=1732014867469; duration=0sec 2024-11-19T11:14:28,078 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:28,078 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:C 2024-11-19T11:14:28,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741878_1054 (size=12151) 2024-11-19T11:14:28,091 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/427d5dc3046b4adfaab4c17a9c767802 2024-11-19T11:14:28,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/c04c2fd9254546e1b8f9c71f7d871594 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/c04c2fd9254546e1b8f9c71f7d871594 2024-11-19T11:14:28,110 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/c04c2fd9254546e1b8f9c71f7d871594, entries=150, sequenceid=173, filesize=11.9 K 2024-11-19T11:14:28,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/b03b4a3a415442f2a7bd95208a52e831 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/b03b4a3a415442f2a7bd95208a52e831 2024-11-19T11:14:28,121 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/b03b4a3a415442f2a7bd95208a52e831, entries=150, sequenceid=173, filesize=11.9 K 2024-11-19T11:14:28,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/427d5dc3046b4adfaab4c17a9c767802 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/427d5dc3046b4adfaab4c17a9c767802 2024-11-19T11:14:28,137 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/427d5dc3046b4adfaab4c17a9c767802, entries=150, sequenceid=173, filesize=11.9 K 2024-11-19T11:14:28,140 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 9cbe0c20892be91e1583baaaf6ed665f in 580ms, sequenceid=173, compaction requested=false 2024-11-19T11:14:28,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:28,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:28,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-19T11:14:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-19T11:14:28,146 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-19T11:14:28,146 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0490 sec 2024-11-19T11:14:28,149 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.0580 sec 2024-11-19T11:14:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T11:14:28,228 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:14:28,228 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:14:28,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-19T11:14:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T11:14:28,230 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:14:28,231 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:14:28,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:14:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T11:14:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:28,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T11:14:28,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:28,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:28,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:28,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:28,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:28,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:28,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/3d2d7a729e2f4675b85a5bc62513ae54 is 50, key is test_row_0/A:col10/1732014867698/Put/seqid=0 2024-11-19T11:14:28,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741879_1055 (size=14541) 2024-11-19T11:14:28,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/3d2d7a729e2f4675b85a5bc62513ae54 2024-11-19T11:14:28,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T11:14:28,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:28,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:28,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:28,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:28,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:28,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:28,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/9179c824fb1e4c9597820ec7a5224c66 is 50, key is test_row_0/B:col10/1732014867698/Put/seqid=0 2024-11-19T11:14:28,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741880_1056 (size=12151) 2024-11-19T11:14:28,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014928429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:28,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T11:14:28,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:28,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:28,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:28,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014928538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:28,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:28,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:28,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:28,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:28,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T11:14:28,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T11:14:28,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:28,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:28,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:28,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:28,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:28,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014928747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:28,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:28,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:28,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:28,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/9179c824fb1e4c9597820ec7a5224c66 2024-11-19T11:14:28,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T11:14:28,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:28,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:28,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:28,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:28,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T11:14:28,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/b286eeb799554344974a608a657c3504 is 50, key is test_row_0/C:col10/1732014867698/Put/seqid=0 2024-11-19T11:14:28,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741881_1057 (size=12151) 2024-11-19T11:14:29,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T11:14:29,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:29,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:29,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:29,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:29,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:29,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014929057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:29,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T11:14:29,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:29,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:29,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:29,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:29,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:29,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/b286eeb799554344974a608a657c3504 2024-11-19T11:14:29,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/3d2d7a729e2f4675b85a5bc62513ae54 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/3d2d7a729e2f4675b85a5bc62513ae54 2024-11-19T11:14:29,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/3d2d7a729e2f4675b85a5bc62513ae54, entries=200, sequenceid=192, filesize=14.2 K 2024-11-19T11:14:29,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/9179c824fb1e4c9597820ec7a5224c66 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/9179c824fb1e4c9597820ec7a5224c66 2024-11-19T11:14:29,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/9179c824fb1e4c9597820ec7a5224c66, entries=150, sequenceid=192, filesize=11.9 K 2024-11-19T11:14:29,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/b286eeb799554344974a608a657c3504 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/b286eeb799554344974a608a657c3504 2024-11-19T11:14:29,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/b286eeb799554344974a608a657c3504, entries=150, sequenceid=192, filesize=11.9 K 2024-11-19T11:14:29,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 9cbe0c20892be91e1583baaaf6ed665f in 956ms, sequenceid=192, compaction requested=true 2024-11-19T11:14:29,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:29,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:14:29,316 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:29,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:29,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:14:29,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:29,317 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:29,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:14:29,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:29,319 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:29,319 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:29,319 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/B is initiating minor compaction (all files) 2024-11-19T11:14:29,319 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/A is initiating minor compaction (all files) 2024-11-19T11:14:29,319 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/B in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:29,319 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/A in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:29,319 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/6400a5d6fd5d4209911c99c56a691152, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/c04c2fd9254546e1b8f9c71f7d871594, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/3d2d7a729e2f4675b85a5bc62513ae54] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=38.3 K 2024-11-19T11:14:29,319 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/7fe5a07da1e9437d9d2d8fb50814590c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/b03b4a3a415442f2a7bd95208a52e831, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/9179c824fb1e4c9597820ec7a5224c66] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=36.0 K 2024-11-19T11:14:29,320 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6400a5d6fd5d4209911c99c56a691152, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732014865706 2024-11-19T11:14:29,320 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fe5a07da1e9437d9d2d8fb50814590c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732014865706 2024-11-19T11:14:29,320 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting c04c2fd9254546e1b8f9c71f7d871594, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732014866942 2024-11-19T11:14:29,321 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d2d7a729e2f4675b85a5bc62513ae54, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732014867652 2024-11-19T11:14:29,321 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting b03b4a3a415442f2a7bd95208a52e831, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732014866942 2024-11-19T11:14:29,321 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 9179c824fb1e4c9597820ec7a5224c66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732014867652 2024-11-19T11:14:29,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T11:14:29,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:29,330 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T11:14:29,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:29,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:29,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:29,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:29,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:29,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:29,347 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#B#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:29,347 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/8157f81cc8ff475393a8bbc418af52b1 is 50, key is test_row_0/B:col10/1732014867698/Put/seqid=0 2024-11-19T11:14:29,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/9db9c472f5414768bcc1b5249f9a3f8b is 50, key is test_row_0/A:col10/1732014868406/Put/seqid=0 2024-11-19T11:14:29,353 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#A#compaction#47 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:29,354 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/0f7e5c1c1c8d471db114c38b1dae6007 is 50, key is test_row_0/A:col10/1732014867698/Put/seqid=0 2024-11-19T11:14:29,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741882_1058 (size=12151) 2024-11-19T11:14:29,364 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/9db9c472f5414768bcc1b5249f9a3f8b 2024-11-19T11:14:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T11:14:29,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741883_1059 (size=12663) 2024-11-19T11:14:29,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741884_1060 (size=12663) 2024-11-19T11:14:29,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/c29569d5775949268ece474c65e64ac4 is 50, key is test_row_0/B:col10/1732014868406/Put/seqid=0 2024-11-19T11:14:29,380 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/0f7e5c1c1c8d471db114c38b1dae6007 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/0f7e5c1c1c8d471db114c38b1dae6007 2024-11-19T11:14:29,389 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/A of 9cbe0c20892be91e1583baaaf6ed665f into 0f7e5c1c1c8d471db114c38b1dae6007(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:29,389 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:29,389 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/A, priority=13, startTime=1732014869316; duration=0sec 2024-11-19T11:14:29,389 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:29,389 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:A 2024-11-19T11:14:29,389 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:29,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741885_1061 (size=12151) 2024-11-19T11:14:29,391 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:29,391 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/C is initiating minor compaction (all files) 2024-11-19T11:14:29,391 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/C in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:29,391 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/c29569d5775949268ece474c65e64ac4 2024-11-19T11:14:29,391 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/23b039d7678741889d9fa12ce6f6480c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/427d5dc3046b4adfaab4c17a9c767802, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/b286eeb799554344974a608a657c3504] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=36.0 K 2024-11-19T11:14:29,392 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23b039d7678741889d9fa12ce6f6480c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732014865706 2024-11-19T11:14:29,394 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 427d5dc3046b4adfaab4c17a9c767802, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732014866942 2024-11-19T11:14:29,395 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting b286eeb799554344974a608a657c3504, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732014867652 2024-11-19T11:14:29,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/1a36b94a44574b9c9bf127af9f072389 is 50, key is test_row_0/C:col10/1732014868406/Put/seqid=0 2024-11-19T11:14:29,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741886_1062 (size=12151) 2024-11-19T11:14:29,424 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/1a36b94a44574b9c9bf127af9f072389 2024-11-19T11:14:29,425 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#C#compaction#50 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:29,426 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/284ac5d06fa248eb8c7f1a9b33df86bd is 50, key is test_row_0/C:col10/1732014867698/Put/seqid=0 2024-11-19T11:14:29,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/9db9c472f5414768bcc1b5249f9a3f8b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/9db9c472f5414768bcc1b5249f9a3f8b 2024-11-19T11:14:29,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741887_1063 (size=12663) 2024-11-19T11:14:29,444 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/9db9c472f5414768bcc1b5249f9a3f8b, entries=150, sequenceid=207, filesize=11.9 K 2024-11-19T11:14:29,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/c29569d5775949268ece474c65e64ac4 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/c29569d5775949268ece474c65e64ac4 2024-11-19T11:14:29,452 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/c29569d5775949268ece474c65e64ac4, entries=150, sequenceid=207, filesize=11.9 K 2024-11-19T11:14:29,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/1a36b94a44574b9c9bf127af9f072389 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/1a36b94a44574b9c9bf127af9f072389 2024-11-19T11:14:29,464 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/1a36b94a44574b9c9bf127af9f072389, entries=150, sequenceid=207, filesize=11.9 K 2024-11-19T11:14:29,466 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 9cbe0c20892be91e1583baaaf6ed665f in 135ms, sequenceid=207, compaction requested=false 2024-11-19T11:14:29,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:29,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:29,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-19T11:14:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-19T11:14:29,472 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-19T11:14:29,472 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2370 sec 2024-11-19T11:14:29,477 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.2470 sec 2024-11-19T11:14:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:29,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T11:14:29,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:29,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:29,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:29,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:29,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:29,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:29,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/78422ee05be642fe95331c5c45e1f701 is 50, key is test_row_0/A:col10/1732014869601/Put/seqid=0 2024-11-19T11:14:29,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741888_1064 (size=14541) 2024-11-19T11:14:29,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/78422ee05be642fe95331c5c45e1f701 2024-11-19T11:14:29,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/4ddefab20c924761a13f938212abf0d1 is 50, key is test_row_0/B:col10/1732014869601/Put/seqid=0 2024-11-19T11:14:29,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741889_1065 (size=12151) 2024-11-19T11:14:29,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/4ddefab20c924761a13f938212abf0d1 2024-11-19T11:14:29,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/0bbde82016994f3fadf88c28fda94e6e is 50, key is test_row_0/C:col10/1732014869601/Put/seqid=0 2024-11-19T11:14:29,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741890_1066 (size=12151) 2024-11-19T11:14:29,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/0bbde82016994f3fadf88c28fda94e6e 2024-11-19T11:14:29,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/78422ee05be642fe95331c5c45e1f701 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/78422ee05be642fe95331c5c45e1f701 2024-11-19T11:14:29,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/78422ee05be642fe95331c5c45e1f701, entries=200, sequenceid=218, filesize=14.2 K 2024-11-19T11:14:29,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/4ddefab20c924761a13f938212abf0d1 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/4ddefab20c924761a13f938212abf0d1 2024-11-19T11:14:29,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/4ddefab20c924761a13f938212abf0d1, entries=150, sequenceid=218, filesize=11.9 K 2024-11-19T11:14:29,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/0bbde82016994f3fadf88c28fda94e6e as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/0bbde82016994f3fadf88c28fda94e6e 2024-11-19T11:14:29,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/0bbde82016994f3fadf88c28fda94e6e, entries=150, sequenceid=218, filesize=11.9 K 2024-11-19T11:14:29,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=114.05 KB/116790 for 9cbe0c20892be91e1583baaaf6ed665f in 123ms, sequenceid=218, compaction requested=true 2024-11-19T11:14:29,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:29,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:A, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:14:29,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:29,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:B, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:14:29,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T11:14:29,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:14:29,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-19T11:14:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:29,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732014869646, max timestamp=1732014869726Num uniques 859; 2024-11-19T11:14:29,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732014869646, max timestamp=1732014869726Num uniques 859; 2024-11-19T11:14:29,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T11:14:29,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T11:14:29,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=62.62 KB heapSize=174.33 KB 2024-11-19T11:14:29,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:29,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014869692, max timestamp=1732014869726Num uniques 859; 2024-11-19T11:14:29,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:14:29,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:29,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:29,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:29,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:29,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:29,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/55aa0f703bcd4b0b83ec5e82d2dd5581 is 50, key is test_row_0/A:col10/1732014869738/Put/seqid=0 2024-11-19T11:14:29,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741891_1067 (size=14541) 2024-11-19T11:14:29,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/55aa0f703bcd4b0b83ec5e82d2dd5581 2024-11-19T11:14:29,793 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/8157f81cc8ff475393a8bbc418af52b1 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/8157f81cc8ff475393a8bbc418af52b1 2024-11-19T11:14:29,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/d5489b3911d24479a21ca3bb99d9e8c9 is 50, key is test_row_0/B:col10/1732014869738/Put/seqid=0 2024-11-19T11:14:29,803 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/B of 9cbe0c20892be91e1583baaaf6ed665f into 8157f81cc8ff475393a8bbc418af52b1(size=12.4 K), total size for store is 36.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:29,803 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:29,803 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/B, priority=13, startTime=1732014869316; duration=0sec 2024-11-19T11:14:29,803 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-19T11:14:29,803 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:B 2024-11-19T11:14:29,804 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:B 2024-11-19T11:14:29,804 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-19T11:14:29,807 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-19T11:14:29,807 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-19T11:14:29,807 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. because compaction request was cancelled 2024-11-19T11:14:29,807 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:C 2024-11-19T11:14:29,807 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:29,809 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:29,809 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/B is initiating minor compaction (all files) 2024-11-19T11:14:29,809 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/B in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:29,810 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/8157f81cc8ff475393a8bbc418af52b1, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/c29569d5775949268ece474c65e64ac4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/4ddefab20c924761a13f938212abf0d1] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=36.1 K 2024-11-19T11:14:29,810 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8157f81cc8ff475393a8bbc418af52b1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732014867652 2024-11-19T11:14:29,811 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting c29569d5775949268ece474c65e64ac4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732014868406 2024-11-19T11:14:29,811 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ddefab20c924761a13f938212abf0d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732014869584 2024-11-19T11:14:29,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741892_1068 (size=12151) 2024-11-19T11:14:29,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/d5489b3911d24479a21ca3bb99d9e8c9 2024-11-19T11:14:29,826 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#B#compaction#56 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:29,826 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/d83068855951415a97f403b466e14537 is 50, key is test_row_0/B:col10/1732014869601/Put/seqid=0 2024-11-19T11:14:29,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/bd66ca8564824806ac0a2d0d511bbe33 is 50, key is test_row_0/C:col10/1732014869738/Put/seqid=0 2024-11-19T11:14:29,852 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/284ac5d06fa248eb8c7f1a9b33df86bd as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/284ac5d06fa248eb8c7f1a9b33df86bd 2024-11-19T11:14:29,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741893_1069 (size=12765) 2024-11-19T11:14:29,860 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/C of 9cbe0c20892be91e1583baaaf6ed665f into 284ac5d06fa248eb8c7f1a9b33df86bd(size=12.4 K), total size for store is 36.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:29,860 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:29,860 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/C, priority=13, startTime=1732014869317; duration=0sec 2024-11-19T11:14:29,861 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:29,861 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:C 2024-11-19T11:14:29,861 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:29,862 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:29,862 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/A is initiating minor compaction (all files) 2024-11-19T11:14:29,863 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/A in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:29,863 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/0f7e5c1c1c8d471db114c38b1dae6007, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/9db9c472f5414768bcc1b5249f9a3f8b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/78422ee05be642fe95331c5c45e1f701] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=38.4 K 2024-11-19T11:14:29,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741894_1070 (size=12151) 2024-11-19T11:14:29,864 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f7e5c1c1c8d471db114c38b1dae6007, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732014867652 2024-11-19T11:14:29,864 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9db9c472f5414768bcc1b5249f9a3f8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732014868406 2024-11-19T11:14:29,865 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78422ee05be642fe95331c5c45e1f701, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732014869584 2024-11-19T11:14:29,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/bd66ca8564824806ac0a2d0d511bbe33 2024-11-19T11:14:29,874 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/d83068855951415a97f403b466e14537 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/d83068855951415a97f403b466e14537 2024-11-19T11:14:29,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/55aa0f703bcd4b0b83ec5e82d2dd5581 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/55aa0f703bcd4b0b83ec5e82d2dd5581 2024-11-19T11:14:29,887 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/B of 9cbe0c20892be91e1583baaaf6ed665f into d83068855951415a97f403b466e14537(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:29,887 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:29,887 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/B, priority=13, startTime=1732014869727; duration=0sec 2024-11-19T11:14:29,887 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:29,887 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:B 2024-11-19T11:14:29,888 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#A#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:29,889 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/529b71a91142424cbd2e57d09862e342 is 50, key is test_row_0/A:col10/1732014869601/Put/seqid=0 2024-11-19T11:14:29,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/55aa0f703bcd4b0b83ec5e82d2dd5581, entries=200, sequenceid=240, filesize=14.2 K 2024-11-19T11:14:29,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014869854, max timestamp=1732014869877Num uniques 859; 2024-11-19T11:14:29,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/d5489b3911d24479a21ca3bb99d9e8c9 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/d5489b3911d24479a21ca3bb99d9e8c9 2024-11-19T11:14:29,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:14:29,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014869854, max timestamp=1732014869877Num uniques 859; 2024-11-19T11:14:29,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:14:29,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/d5489b3911d24479a21ca3bb99d9e8c9, entries=150, sequenceid=240, filesize=11.9 K 2024-11-19T11:14:29,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014869854, max timestamp=1732014869877Num uniques 859; 2024-11-19T11:14:29,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:14:29,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/bd66ca8564824806ac0a2d0d511bbe33 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/bd66ca8564824806ac0a2d0d511bbe33 2024-11-19T11:14:29,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741895_1071 (size=12765) 2024-11-19T11:14:29,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/bd66ca8564824806ac0a2d0d511bbe33, entries=150, sequenceid=240, filesize=11.9 K 2024-11-19T11:14:29,916 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/529b71a91142424cbd2e57d09862e342 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/529b71a91142424cbd2e57d09862e342 2024-11-19T11:14:29,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~38.02 KB/38930, heapSize ~96.27 KB/98576, currentSize=40.25 KB/41220 for 9cbe0c20892be91e1583baaaf6ed665f in 178ms, sequenceid=240, compaction requested=true 2024-11-19T11:14:29,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:29,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:14:29,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:29,918 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-19T11:14:29,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:14:29,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:29,919 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-19T11:14:29,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:14:29,919 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-19T11:14:29,919 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. because compaction request was cancelled 2024-11-19T11:14:29,919 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:A 2024-11-19T11:14:29,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T11:14:29,919 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:14:29,921 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:14:29,921 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/C is initiating minor compaction (all files) 2024-11-19T11:14:29,921 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/C in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:29,921 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/284ac5d06fa248eb8c7f1a9b33df86bd, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/1a36b94a44574b9c9bf127af9f072389, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/0bbde82016994f3fadf88c28fda94e6e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/bd66ca8564824806ac0a2d0d511bbe33] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=48.0 K 2024-11-19T11:14:29,922 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 284ac5d06fa248eb8c7f1a9b33df86bd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732014867652 2024-11-19T11:14:29,922 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a36b94a44574b9c9bf127af9f072389, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732014868406 2024-11-19T11:14:29,924 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bbde82016994f3fadf88c28fda94e6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732014869584 2024-11-19T11:14:29,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:29,925 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting bd66ca8564824806ac0a2d0d511bbe33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732014869716 2024-11-19T11:14:29,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-19T11:14:29,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:29,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:29,928 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/A of 9cbe0c20892be91e1583baaaf6ed665f into 529b71a91142424cbd2e57d09862e342(size=12.5 K), total size for store is 26.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:29,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:29,928 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:29,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:29,928 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/A, priority=13, startTime=1732014869727; duration=0sec 2024-11-19T11:14:29,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:29,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:29,929 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:29,929 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:A 2024-11-19T11:14:29,929 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-19T11:14:29,930 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-19T11:14:29,930 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-19T11:14:29,931 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. because compaction request was cancelled 2024-11-19T11:14:29,931 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:B 2024-11-19T11:14:29,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/750553ba3e4047b4acfa89e3f2b5c199 is 50, key is test_row_0/A:col10/1732014869922/Put/seqid=0 2024-11-19T11:14:29,941 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#C#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:29,942 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/899a49a5411c4f7f8bc9931d9b87653f is 50, key is test_row_0/C:col10/1732014869738/Put/seqid=0 2024-11-19T11:14:29,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741896_1072 (size=17181) 2024-11-19T11:14:29,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741897_1073 (size=12799) 2024-11-19T11:14:29,979 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/899a49a5411c4f7f8bc9931d9b87653f as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/899a49a5411c4f7f8bc9931d9b87653f 2024-11-19T11:14:29,987 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/C of 9cbe0c20892be91e1583baaaf6ed665f into 899a49a5411c4f7f8bc9931d9b87653f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:29,987 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:29,987 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/C, priority=12, startTime=1732014869919; duration=0sec 2024-11-19T11:14:29,987 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:29,987 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:C 2024-11-19T11:14:30,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014869963, max timestamp=1732014870043Num uniques 859; 2024-11-19T11:14:30,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014869963, max timestamp=1732014870043Num uniques 859; 2024-11-19T11:14:30,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:14:30,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014869963, max timestamp=1732014870043Num uniques 859; 2024-11-19T11:14:30,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:14:30,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:14:30,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014930143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:30,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:30,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:30,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:30,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014930257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:30,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:30,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:30,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:30,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/750553ba3e4047b4acfa89e3f2b5c199 2024-11-19T11:14:30,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/8e94f57afade48b490dc849d729cc199 is 50, key is test_row_0/B:col10/1732014869922/Put/seqid=0 2024-11-19T11:14:30,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741898_1074 (size=12301) 2024-11-19T11:14:30,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T11:14:30,377 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:14:30,378 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:14:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-19T11:14:30,380 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:14:30,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T11:14:30,384 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:14:30,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:14:30,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:30,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014930467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:30,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:30,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:30,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T11:14:30,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T11:14:30,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:30,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:30,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:30,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:30,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:30,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T11:14:30,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:30,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:30,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:30,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:30,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T11:14:30,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/8e94f57afade48b490dc849d729cc199 2024-11-19T11:14:30,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014930778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:30,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/90b635d8f4db4076ac9589c8404c5cdd is 50, key is test_row_0/C:col10/1732014869922/Put/seqid=0 2024-11-19T11:14:30,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741899_1075 (size=12301) 2024-11-19T11:14:30,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T11:14:30,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:30,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:30,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:30,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:30,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:30,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T11:14:30,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:30,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:30,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:30,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:30,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T11:14:31,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T11:14:31,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:31,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:31,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:31,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:31,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:31,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/90b635d8f4db4076ac9589c8404c5cdd 2024-11-19T11:14:31,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/750553ba3e4047b4acfa89e3f2b5c199 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/750553ba3e4047b4acfa89e3f2b5c199 2024-11-19T11:14:31,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/750553ba3e4047b4acfa89e3f2b5c199, entries=250, sequenceid=269, filesize=16.8 K 2024-11-19T11:14:31,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/8e94f57afade48b490dc849d729cc199 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/8e94f57afade48b490dc849d729cc199 2024-11-19T11:14:31,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/8e94f57afade48b490dc849d729cc199, entries=150, sequenceid=269, filesize=12.0 K 2024-11-19T11:14:31,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/90b635d8f4db4076ac9589c8404c5cdd as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/90b635d8f4db4076ac9589c8404c5cdd 2024-11-19T11:14:31,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/90b635d8f4db4076ac9589c8404c5cdd, entries=150, sequenceid=269, filesize=12.0 K 2024-11-19T11:14:31,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for 9cbe0c20892be91e1583baaaf6ed665f in 1304ms, sequenceid=269, compaction requested=true 2024-11-19T11:14:31,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:31,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:14:31,232 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:31,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:31,232 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:31,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:14:31,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:31,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:14:31,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:31,233 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44487 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:31,233 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:31,233 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/B is initiating minor compaction (all files) 2024-11-19T11:14:31,233 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/A is initiating minor compaction (all files) 2024-11-19T11:14:31,233 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/B in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:31,233 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/A in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:31,233 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/529b71a91142424cbd2e57d09862e342, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/55aa0f703bcd4b0b83ec5e82d2dd5581, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/750553ba3e4047b4acfa89e3f2b5c199] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=43.4 K 2024-11-19T11:14:31,233 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/d83068855951415a97f403b466e14537, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/d5489b3911d24479a21ca3bb99d9e8c9, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/8e94f57afade48b490dc849d729cc199] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=36.3 K 2024-11-19T11:14:31,234 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting d83068855951415a97f403b466e14537, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732014869584 2024-11-19T11:14:31,234 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 529b71a91142424cbd2e57d09862e342, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732014869584 2024-11-19T11:14:31,234 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting d5489b3911d24479a21ca3bb99d9e8c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732014869716 2024-11-19T11:14:31,234 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55aa0f703bcd4b0b83ec5e82d2dd5581, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732014869711 2024-11-19T11:14:31,235 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e94f57afade48b490dc849d729cc199, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732014869877 2024-11-19T11:14:31,235 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 750553ba3e4047b4acfa89e3f2b5c199, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732014869854 2024-11-19T11:14:31,248 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#B#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:31,248 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/8dbd464e62f042a0911ba8bf7d11b627 is 50, key is test_row_0/B:col10/1732014869922/Put/seqid=0 2024-11-19T11:14:31,251 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#A#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:31,252 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/86541d5799414f6d8be8c6a70c1eb3e0 is 50, key is test_row_0/A:col10/1732014869922/Put/seqid=0 2024-11-19T11:14:31,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741901_1077 (size=13017) 2024-11-19T11:14:31,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741900_1076 (size=13017) 2024-11-19T11:14:31,279 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/86541d5799414f6d8be8c6a70c1eb3e0 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/86541d5799414f6d8be8c6a70c1eb3e0 2024-11-19T11:14:31,284 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/8dbd464e62f042a0911ba8bf7d11b627 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/8dbd464e62f042a0911ba8bf7d11b627 2024-11-19T11:14:31,295 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/B of 9cbe0c20892be91e1583baaaf6ed665f into 8dbd464e62f042a0911ba8bf7d11b627(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:31,295 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:31,295 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/B, priority=13, startTime=1732014871232; duration=0sec 2024-11-19T11:14:31,295 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:31,295 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:B 2024-11-19T11:14:31,295 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-19T11:14:31,296 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-19T11:14:31,296 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-19T11:14:31,296 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. because compaction request was cancelled 2024-11-19T11:14:31,296 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:C 2024-11-19T11:14:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:31,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-19T11:14:31,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:31,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T11:14:31,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:31,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T11:14:31,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:31,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T11:14:31,298 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/A of 9cbe0c20892be91e1583baaaf6ed665f into 86541d5799414f6d8be8c6a70c1eb3e0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:31,298 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:31,298 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/A, priority=13, startTime=1732014871232; duration=0sec 2024-11-19T11:14:31,302 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:31,302 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:A 2024-11-19T11:14:31,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T11:14:31,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/ccdaec296a54436cbe5d35630296022a is 50, key is test_row_0/A:col10/1732014870134/Put/seqid=0 2024-11-19T11:14:31,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:31,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:31,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:31,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:31,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:31,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741902_1078 (size=14741) 2024-11-19T11:14:31,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014931353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:31,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:31,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:31,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:31,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T11:14:31,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:31,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:31,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:31,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:31,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:31,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014931468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T11:14:31,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T11:14:31,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:31,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:31,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:31,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:31,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:31,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014931677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:31,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:31,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:31,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:31,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/ccdaec296a54436cbe5d35630296022a 2024-11-19T11:14:31,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/5999f47618124442bee622d407ca465b is 50, key is test_row_0/B:col10/1732014870134/Put/seqid=0 2024-11-19T11:14:31,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741903_1079 (size=12301) 2024-11-19T11:14:31,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T11:14:31,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:31,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:31,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:31,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:31,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:31,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:31,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T11:14:31,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:31,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:31,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:31,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:31,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:31,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:31,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:31,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014931988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:31,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:31,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:31,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:32,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:32,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46270 deadline: 1732014932067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:32,068 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_1' in 9cbe0c20892be91e1583baaaf6ed665f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:14:32,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:32,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:32,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:32,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T11:14:32,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:32,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:32,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:32,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:32,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:32,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:32,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46260 deadline: 1732014932078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:32,083 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_2' in 9cbe0c20892be91e1583baaaf6ed665f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8274 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:14:32,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:32,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:32,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:32,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:32,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46264 deadline: 1732014932087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:32,088 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_1' in 9cbe0c20892be91e1583baaaf6ed665f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:14:32,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:32,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:32,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:32,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:32,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46208 deadline: 1732014932097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:32,099 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_2' in 9cbe0c20892be91e1583baaaf6ed665f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:14:32,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:32,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:32,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:32,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/5999f47618124442bee622d407ca465b 2024-11-19T11:14:32,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/0e39fb045de848269219f6d3748bf2d1 is 50, key is test_row_0/C:col10/1732014870134/Put/seqid=0 2024-11-19T11:14:32,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741904_1080 (size=12301) 2024-11-19T11:14:32,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/0e39fb045de848269219f6d3748bf2d1 2024-11-19T11:14:32,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/ccdaec296a54436cbe5d35630296022a as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/ccdaec296a54436cbe5d35630296022a 2024-11-19T11:14:32,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/ccdaec296a54436cbe5d35630296022a, entries=200, sequenceid=310, filesize=14.4 K 2024-11-19T11:14:32,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/5999f47618124442bee622d407ca465b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/5999f47618124442bee622d407ca465b 2024-11-19T11:14:32,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/5999f47618124442bee622d407ca465b, entries=150, sequenceid=310, filesize=12.0 K 2024-11-19T11:14:32,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/0e39fb045de848269219f6d3748bf2d1 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/0e39fb045de848269219f6d3748bf2d1 2024-11-19T11:14:32,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/0e39fb045de848269219f6d3748bf2d1, entries=150, sequenceid=310, filesize=12.0 K 2024-11-19T11:14:32,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 9cbe0c20892be91e1583baaaf6ed665f in 906ms, sequenceid=310, compaction requested=true 2024-11-19T11:14:32,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:32,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:14:32,203 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-19T11:14:32,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:32,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:14:32,203 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-19T11:14:32,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:32,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:14:32,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:32,204 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-19T11:14:32,204 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-19T11:14:32,204 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-19T11:14:32,204 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. because compaction request was cancelled 2024-11-19T11:14:32,204 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-19T11:14:32,204 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:A 2024-11-19T11:14:32,204 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. because compaction request was cancelled 2024-11-19T11:14:32,204 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:32,204 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:B 2024-11-19T11:14:32,205 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:32,205 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/C is initiating minor compaction (all files) 2024-11-19T11:14:32,205 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/C in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:32,205 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/899a49a5411c4f7f8bc9931d9b87653f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/90b635d8f4db4076ac9589c8404c5cdd, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/0e39fb045de848269219f6d3748bf2d1] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=36.5 K 2024-11-19T11:14:32,206 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 899a49a5411c4f7f8bc9931d9b87653f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732014869716 2024-11-19T11:14:32,206 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90b635d8f4db4076ac9589c8404c5cdd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732014869877 2024-11-19T11:14:32,206 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e39fb045de848269219f6d3748bf2d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732014870134 2024-11-19T11:14:32,216 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#C#compaction#68 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:32,218 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/cba72ceb6b764324bd149f7145f0dffd is 50, key is test_row_0/C:col10/1732014870134/Put/seqid=0 2024-11-19T11:14:32,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T11:14:32,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:32,231 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-19T11:14:32,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:32,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:32,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:32,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:32,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:32,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:32,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/4051293baa7447d7b33537aaebe3a5db is 50, key is test_row_0/A:col10/1732014871352/Put/seqid=0 2024-11-19T11:14:32,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741905_1081 (size=13051) 2024-11-19T11:14:32,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741906_1082 (size=9857) 2024-11-19T11:14:32,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:32,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:32,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T11:14:32,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014872550, max timestamp=1732014872574Num uniques 859; 2024-11-19T11:14:32,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:14:32,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014872550, max timestamp=1732014872574Num uniques 859; 2024-11-19T11:14:32,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:14:32,644 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/4051293baa7447d7b33537aaebe3a5db 2024-11-19T11:14:32,649 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/cba72ceb6b764324bd149f7145f0dffd as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/cba72ceb6b764324bd149f7145f0dffd 2024-11-19T11:14:32,657 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/C of 9cbe0c20892be91e1583baaaf6ed665f into cba72ceb6b764324bd149f7145f0dffd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:32,657 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:32,657 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/C, priority=13, startTime=1732014872203; duration=0sec 2024-11-19T11:14:32,658 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:32,659 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:C 2024-11-19T11:14:32,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/3e12d2353ed645e4ac55697b33a47ea0 is 50, key is test_row_0/B:col10/1732014871352/Put/seqid=0 2024-11-19T11:14:32,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741907_1083 (size=9857) 2024-11-19T11:14:32,674 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/3e12d2353ed645e4ac55697b33a47ea0 2024-11-19T11:14:32,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014932669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:32,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/6c9de9fa735742f6a3c589c534516b32 is 50, key is test_row_0/C:col10/1732014871352/Put/seqid=0 2024-11-19T11:14:32,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741908_1084 (size=9857) 2024-11-19T11:14:32,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:32,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014932777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:32,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:32,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:32,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:32,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014932987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:33,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:33,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:33,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:33,128 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/6c9de9fa735742f6a3c589c534516b32 2024-11-19T11:14:33,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/4051293baa7447d7b33537aaebe3a5db as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/4051293baa7447d7b33537aaebe3a5db 2024-11-19T11:14:33,141 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/4051293baa7447d7b33537aaebe3a5db, entries=100, sequenceid=319, filesize=9.6 K 2024-11-19T11:14:33,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/3e12d2353ed645e4ac55697b33a47ea0 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/3e12d2353ed645e4ac55697b33a47ea0 2024-11-19T11:14:33,150 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/3e12d2353ed645e4ac55697b33a47ea0, entries=100, sequenceid=319, filesize=9.6 K 2024-11-19T11:14:33,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/6c9de9fa735742f6a3c589c534516b32 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/6c9de9fa735742f6a3c589c534516b32 2024-11-19T11:14:33,159 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/6c9de9fa735742f6a3c589c534516b32, entries=100, sequenceid=319, filesize=9.6 K 2024-11-19T11:14:33,161 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=138.65 KB/141980 for 9cbe0c20892be91e1583baaaf6ed665f in 929ms, sequenceid=319, compaction requested=true 2024-11-19T11:14:33,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:33,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:33,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-19T11:14:33,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-19T11:14:33,165 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-19T11:14:33,165 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7780 sec 2024-11-19T11:14:33,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.7890 sec 2024-11-19T11:14:33,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:33,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=145.36 KB heapSize=423.16 KB 2024-11-19T11:14:33,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:33,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:33,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:33,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:33,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:33,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:33,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/4357a63cf66849fd8b90cc4e55890e7d is 50, key is test_row_0/A:col10/1732014872645/Put/seqid=0 2024-11-19T11:14:33,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741909_1085 (size=14741) 2024-11-19T11:14:33,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:33,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014933332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:33,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:33,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:33,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:33,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:33,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014933437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:33,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014933657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:33,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:33,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:33,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:33,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=69.33 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/4357a63cf66849fd8b90cc4e55890e7d 2024-11-19T11:14:33,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/e24e1ac8473e44c295a8082ba575b1cb is 50, key is test_row_0/B:col10/1732014872645/Put/seqid=0 2024-11-19T11:14:33,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741910_1086 (size=12301) 2024-11-19T11:14:33,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:33,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014933968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:33,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:33,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:33,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:34,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/e24e1ac8473e44c295a8082ba575b1cb 2024-11-19T11:14:34,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/f17ed6d1ac984a9da079f46827e4f1b3 is 50, key is test_row_0/C:col10/1732014872645/Put/seqid=0 2024-11-19T11:14:34,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741911_1087 (size=12301) 2024-11-19T11:14:34,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/f17ed6d1ac984a9da079f46827e4f1b3 2024-11-19T11:14:34,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/4357a63cf66849fd8b90cc4e55890e7d as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/4357a63cf66849fd8b90cc4e55890e7d 2024-11-19T11:14:34,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/4357a63cf66849fd8b90cc4e55890e7d, entries=200, sequenceid=354, filesize=14.4 K 2024-11-19T11:14:34,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/e24e1ac8473e44c295a8082ba575b1cb as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/e24e1ac8473e44c295a8082ba575b1cb 2024-11-19T11:14:34,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/e24e1ac8473e44c295a8082ba575b1cb, entries=150, sequenceid=354, filesize=12.0 K 2024-11-19T11:14:34,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/f17ed6d1ac984a9da079f46827e4f1b3 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/f17ed6d1ac984a9da079f46827e4f1b3 2024-11-19T11:14:34,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/f17ed6d1ac984a9da079f46827e4f1b3, entries=150, sequenceid=354, filesize=12.0 K 2024-11-19T11:14:34,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~145.36 KB/148850, heapSize ~423.11 KB/433264, currentSize=33.54 KB/34350 for 9cbe0c20892be91e1583baaaf6ed665f in 919ms, sequenceid=354, compaction requested=true 2024-11-19T11:14:34,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:34,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:14:34,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:34,228 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:14:34,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:14:34,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:34,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:14:34,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T11:14:34,229 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:14:34,230 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52356 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:14:34,230 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/A is initiating minor compaction (all files) 2024-11-19T11:14:34,230 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/A in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:34,230 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/86541d5799414f6d8be8c6a70c1eb3e0, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/ccdaec296a54436cbe5d35630296022a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/4051293baa7447d7b33537aaebe3a5db, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/4357a63cf66849fd8b90cc4e55890e7d] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=51.1 K 2024-11-19T11:14:34,231 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:14:34,231 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/B is initiating minor compaction (all files) 2024-11-19T11:14:34,231 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/B in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:34,231 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/8dbd464e62f042a0911ba8bf7d11b627, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/5999f47618124442bee622d407ca465b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/3e12d2353ed645e4ac55697b33a47ea0, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/e24e1ac8473e44c295a8082ba575b1cb] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=46.4 K 2024-11-19T11:14:34,231 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86541d5799414f6d8be8c6a70c1eb3e0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732014869877 2024-11-19T11:14:34,232 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dbd464e62f042a0911ba8bf7d11b627, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732014869877 2024-11-19T11:14:34,232 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccdaec296a54436cbe5d35630296022a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732014870115 2024-11-19T11:14:34,232 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 5999f47618124442bee622d407ca465b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732014870134 2024-11-19T11:14:34,232 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4051293baa7447d7b33537aaebe3a5db, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732014871345 2024-11-19T11:14:34,233 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e12d2353ed645e4ac55697b33a47ea0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732014871345 2024-11-19T11:14:34,233 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4357a63cf66849fd8b90cc4e55890e7d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732014872645 2024-11-19T11:14:34,233 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting e24e1ac8473e44c295a8082ba575b1cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732014872645 2024-11-19T11:14:34,250 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#B#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:34,250 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/274925480c8b4e029cdf2046b19c1d63 is 50, key is test_row_0/B:col10/1732014872645/Put/seqid=0 2024-11-19T11:14:34,263 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#A#compaction#76 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:34,263 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/3f009cffdff3457981c04c229818a25e is 50, key is test_row_0/A:col10/1732014872645/Put/seqid=0 2024-11-19T11:14:34,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741912_1088 (size=13153) 2024-11-19T11:14:34,282 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/274925480c8b4e029cdf2046b19c1d63 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/274925480c8b4e029cdf2046b19c1d63 2024-11-19T11:14:34,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741913_1089 (size=13153) 2024-11-19T11:14:34,289 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/B of 9cbe0c20892be91e1583baaaf6ed665f into 274925480c8b4e029cdf2046b19c1d63(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:34,289 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:34,289 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/B, priority=12, startTime=1732014874228; duration=0sec 2024-11-19T11:14:34,290 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:34,290 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:B 2024-11-19T11:14:34,290 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:34,292 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:34,293 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/C is initiating minor compaction (all files) 2024-11-19T11:14:34,293 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/C in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:34,293 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/cba72ceb6b764324bd149f7145f0dffd, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/6c9de9fa735742f6a3c589c534516b32, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/f17ed6d1ac984a9da079f46827e4f1b3] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=34.4 K 2024-11-19T11:14:34,293 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting cba72ceb6b764324bd149f7145f0dffd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732014870134 2024-11-19T11:14:34,294 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c9de9fa735742f6a3c589c534516b32, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732014871345 2024-11-19T11:14:34,296 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting f17ed6d1ac984a9da079f46827e4f1b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732014872645 2024-11-19T11:14:34,297 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/3f009cffdff3457981c04c229818a25e as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/3f009cffdff3457981c04c229818a25e 2024-11-19T11:14:34,312 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/A of 9cbe0c20892be91e1583baaaf6ed665f into 3f009cffdff3457981c04c229818a25e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:34,312 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:34,312 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/A, priority=12, startTime=1732014874228; duration=0sec 2024-11-19T11:14:34,313 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:34,313 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:A 2024-11-19T11:14:34,315 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#C#compaction#77 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:34,315 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/a995bac1b1ef4416ad62492fa6b19c36 is 50, key is test_row_0/C:col10/1732014872645/Put/seqid=0 2024-11-19T11:14:34,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741914_1090 (size=13153) 2024-11-19T11:14:34,341 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/a995bac1b1ef4416ad62492fa6b19c36 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/a995bac1b1ef4416ad62492fa6b19c36 2024-11-19T11:14:34,348 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/C of 9cbe0c20892be91e1583baaaf6ed665f into a995bac1b1ef4416ad62492fa6b19c36(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:34,348 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:34,348 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/C, priority=13, startTime=1732014874228; duration=0sec 2024-11-19T11:14:34,348 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:34,348 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:C 2024-11-19T11:14:34,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:34,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T11:14:34,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:34,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:34,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:34,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:34,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:34,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:34,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/46c8d4b715214564ac19b7444cfa0e6e is 50, key is test_row_0/A:col10/1732014874480/Put/seqid=0 2024-11-19T11:14:34,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741915_1091 (size=12301) 2024-11-19T11:14:34,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/46c8d4b715214564ac19b7444cfa0e6e 2024-11-19T11:14:34,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/72cdeec4f42b4f218edd1a46085a19b4 is 50, key is test_row_0/B:col10/1732014874480/Put/seqid=0 2024-11-19T11:14:34,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T11:14:34,548 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:14:34,549 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:14:34,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-19T11:14:34,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T11:14:34,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741916_1092 (size=12301) 2024-11-19T11:14:34,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/72cdeec4f42b4f218edd1a46085a19b4 2024-11-19T11:14:34,562 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:14:34,565 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:14:34,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:14:34,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/74f9c7f25a644f5292832bdcda4b16f2 is 50, key is test_row_0/C:col10/1732014874480/Put/seqid=0 2024-11-19T11:14:34,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741917_1093 (size=12301) 2024-11-19T11:14:34,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/74f9c7f25a644f5292832bdcda4b16f2 2024-11-19T11:14:34,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/46c8d4b715214564ac19b7444cfa0e6e as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/46c8d4b715214564ac19b7444cfa0e6e 2024-11-19T11:14:34,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/46c8d4b715214564ac19b7444cfa0e6e, entries=150, sequenceid=367, filesize=12.0 K 2024-11-19T11:14:34,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732014874567, max timestamp=1732014874611Num uniques 859; 2024-11-19T11:14:34,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/72cdeec4f42b4f218edd1a46085a19b4 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/72cdeec4f42b4f218edd1a46085a19b4 2024-11-19T11:14:34,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T11:14:34,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732014874567, max timestamp=1732014874611Num uniques 859; 2024-11-19T11:14:34,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T11:14:34,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732014874567, max timestamp=1732014874611Num uniques 859; 2024-11-19T11:14:34,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T11:14:34,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/72cdeec4f42b4f218edd1a46085a19b4, entries=150, sequenceid=367, filesize=12.0 K 2024-11-19T11:14:34,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/74f9c7f25a644f5292832bdcda4b16f2 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/74f9c7f25a644f5292832bdcda4b16f2 2024-11-19T11:14:34,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/74f9c7f25a644f5292832bdcda4b16f2, entries=150, sequenceid=367, filesize=12.0 K 2024-11-19T11:14:34,644 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=64.46 KB/66005 for 9cbe0c20892be91e1583baaaf6ed665f in 160ms, sequenceid=367, compaction requested=false 2024-11-19T11:14:34,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:34,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:34,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=71.17 KB heapSize=195.01 KB 2024-11-19T11:14:34,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:34,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:34,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:34,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:34,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:34,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:34,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/a31fdb9334b64f888a71df8ad6ff13ff is 50, key is test_row_0/A:col10/1732014874636/Put/seqid=0 2024-11-19T11:14:34,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741918_1094 (size=14741) 2024-11-19T11:14:34,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/a31fdb9334b64f888a71df8ad6ff13ff 2024-11-19T11:14:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T11:14:34,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/29adcd98f58c44bfbc4543a14c171752 is 50, key is test_row_0/B:col10/1732014874636/Put/seqid=0 2024-11-19T11:14:34,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741919_1095 (size=12301) 2024-11-19T11:14:34,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/29adcd98f58c44bfbc4543a14c171752 2024-11-19T11:14:34,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/8eb46aed68cf436cb8160338ee445045 is 50, key is test_row_0/C:col10/1732014874636/Put/seqid=0 2024-11-19T11:14:34,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T11:14:34,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:34,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:34,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:34,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:34,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:34,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741920_1096 (size=12301) 2024-11-19T11:14:34,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=21.97 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/8eb46aed68cf436cb8160338ee445045 2024-11-19T11:14:34,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/a31fdb9334b64f888a71df8ad6ff13ff as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/a31fdb9334b64f888a71df8ad6ff13ff 2024-11-19T11:14:34,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/a31fdb9334b64f888a71df8ad6ff13ff, entries=200, sequenceid=394, filesize=14.4 K 2024-11-19T11:14:34,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/29adcd98f58c44bfbc4543a14c171752 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/29adcd98f58c44bfbc4543a14c171752 2024-11-19T11:14:34,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 297 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014934748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:34,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:34,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:34,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:34,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/29adcd98f58c44bfbc4543a14c171752, entries=150, sequenceid=394, filesize=12.0 K 2024-11-19T11:14:34,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/8eb46aed68cf436cb8160338ee445045 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/8eb46aed68cf436cb8160338ee445045 2024-11-19T11:14:34,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/8eb46aed68cf436cb8160338ee445045, entries=150, sequenceid=394, filesize=12.0 K 2024-11-19T11:14:34,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.17 KB/72875, heapSize ~194.96 KB/199640, currentSize=107.34 KB/109920 for 9cbe0c20892be91e1583baaaf6ed665f in 113ms, sequenceid=394, compaction requested=true 2024-11-19T11:14:34,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:34,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:14:34,761 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:34,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:34,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:14:34,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:34,761 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:34,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:14:34,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:34,762 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:34,762 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:34,762 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/A is initiating minor compaction (all files) 2024-11-19T11:14:34,762 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/B is initiating minor compaction (all files) 2024-11-19T11:14:34,762 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/A in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:34,763 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/B in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:34,763 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/3f009cffdff3457981c04c229818a25e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/46c8d4b715214564ac19b7444cfa0e6e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/a31fdb9334b64f888a71df8ad6ff13ff] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=39.3 K 2024-11-19T11:14:34,763 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/274925480c8b4e029cdf2046b19c1d63, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/72cdeec4f42b4f218edd1a46085a19b4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/29adcd98f58c44bfbc4543a14c171752] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=36.9 K 2024-11-19T11:14:34,763 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f009cffdff3457981c04c229818a25e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732014872645 2024-11-19T11:14:34,763 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 274925480c8b4e029cdf2046b19c1d63, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732014872645 2024-11-19T11:14:34,764 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46c8d4b715214564ac19b7444cfa0e6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732014873318 2024-11-19T11:14:34,764 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 72cdeec4f42b4f218edd1a46085a19b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732014873318 2024-11-19T11:14:34,764 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting a31fdb9334b64f888a71df8ad6ff13ff, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732014874599 2024-11-19T11:14:34,764 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 29adcd98f58c44bfbc4543a14c171752, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732014874599 2024-11-19T11:14:34,782 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#A#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:34,783 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/3283ecdf1a184e4399899bdcf00c5e8e is 50, key is test_row_0/A:col10/1732014874636/Put/seqid=0 2024-11-19T11:14:34,786 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#B#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:34,787 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/d92363304f62496b9f5ce96ee9fb34dc is 50, key is test_row_0/B:col10/1732014874636/Put/seqid=0 2024-11-19T11:14:34,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741921_1097 (size=13255) 2024-11-19T11:14:34,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741922_1098 (size=13255) 2024-11-19T11:14:34,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:34,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T11:14:34,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:34,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:34,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:34,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:34,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:34,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:34,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/b24f93c994b243f5a9996fc21ce633f3 is 50, key is test_row_0/A:col10/1732014874727/Put/seqid=0 2024-11-19T11:14:34,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T11:14:34,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:34,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:34,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:34,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:34,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T11:14:34,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741923_1099 (size=14741) 2024-11-19T11:14:34,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/b24f93c994b243f5a9996fc21ce633f3 2024-11-19T11:14:34,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 308 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014934906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:34,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/4e2e2fe12ac64df787198a9f1afb1761 is 50, key is test_row_0/B:col10/1732014874727/Put/seqid=0 2024-11-19T11:14:34,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741924_1100 (size=12301) 2024-11-19T11:14:34,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/4e2e2fe12ac64df787198a9f1afb1761 2024-11-19T11:14:34,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/4327adf3d11c451cb6df6b1bd842d433 is 50, key is test_row_0/C:col10/1732014874727/Put/seqid=0 2024-11-19T11:14:34,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741925_1101 (size=12301) 2024-11-19T11:14:34,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/4327adf3d11c451cb6df6b1bd842d433 2024-11-19T11:14:34,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/b24f93c994b243f5a9996fc21ce633f3 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/b24f93c994b243f5a9996fc21ce633f3 2024-11-19T11:14:34,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/b24f93c994b243f5a9996fc21ce633f3, entries=200, sequenceid=414, filesize=14.4 K 2024-11-19T11:14:34,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/4e2e2fe12ac64df787198a9f1afb1761 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/4e2e2fe12ac64df787198a9f1afb1761 2024-11-19T11:14:34,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/4e2e2fe12ac64df787198a9f1afb1761, entries=150, sequenceid=414, filesize=12.0 K 2024-11-19T11:14:34,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/4327adf3d11c451cb6df6b1bd842d433 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/4327adf3d11c451cb6df6b1bd842d433 2024-11-19T11:14:34,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/4327adf3d11c451cb6df6b1bd842d433, entries=150, sequenceid=414, filesize=12.0 K 2024-11-19T11:14:34,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9cbe0c20892be91e1583baaaf6ed665f in 129ms, sequenceid=414, compaction requested=true 2024-11-19T11:14:34,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:34,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:A, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:14:34,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T11:14:34,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:B, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:14:34,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-19T11:14:34,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:14:34,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-19T11:14:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:35,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T11:14:35,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:35,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:35,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:35,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:35,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:35,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:35,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/c39744ba814749758d74727ae57427d7 is 50, key is test_row_0/A:col10/1732014875018/Put/seqid=0 2024-11-19T11:14:35,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T11:14:35,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:35,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:35,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:35,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:35,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:35,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741926_1102 (size=12301) 2024-11-19T11:14:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:35,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/c39744ba814749758d74727ae57427d7 2024-11-19T11:14:35,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/31d5c2f94685411eadb2f03693aa2eba is 50, key is test_row_0/B:col10/1732014875018/Put/seqid=0 2024-11-19T11:14:35,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741927_1103 (size=12301) 2024-11-19T11:14:35,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 326 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014935118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:35,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:35,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:35,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:35,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T11:14:35,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:35,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:35,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:35,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:35,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:35,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:35,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T11:14:35,223 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/d92363304f62496b9f5ce96ee9fb34dc as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/d92363304f62496b9f5ce96ee9fb34dc 2024-11-19T11:14:35,227 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/3283ecdf1a184e4399899bdcf00c5e8e as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/3283ecdf1a184e4399899bdcf00c5e8e 2024-11-19T11:14:35,238 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/B of 9cbe0c20892be91e1583baaaf6ed665f into d92363304f62496b9f5ce96ee9fb34dc(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:35,238 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:35,238 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/B, priority=13, startTime=1732014874761; duration=0sec 2024-11-19T11:14:35,238 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-19T11:14:35,238 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:B 2024-11-19T11:14:35,238 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-19T11:14:35,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 327 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014935238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:35,240 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-19T11:14:35,240 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-19T11:14:35,240 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. because compaction request was cancelled 2024-11-19T11:14:35,240 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:A 2024-11-19T11:14:35,240 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:14:35,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:35,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:35,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:35,243 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:14:35,243 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/C is initiating minor compaction (all files) 2024-11-19T11:14:35,243 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/C in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:35,243 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/A of 9cbe0c20892be91e1583baaaf6ed665f into 3283ecdf1a184e4399899bdcf00c5e8e(size=12.9 K), total size for store is 27.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:35,243 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:35,243 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/a995bac1b1ef4416ad62492fa6b19c36, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/74f9c7f25a644f5292832bdcda4b16f2, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/8eb46aed68cf436cb8160338ee445045, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/4327adf3d11c451cb6df6b1bd842d433] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=48.9 K 2024-11-19T11:14:35,243 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/A, priority=13, startTime=1732014874761; duration=0sec 2024-11-19T11:14:35,243 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T11:14:35,243 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:A 2024-11-19T11:14:35,244 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:B 2024-11-19T11:14:35,244 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-19T11:14:35,244 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-19T11:14:35,244 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting a995bac1b1ef4416ad62492fa6b19c36, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732014872645 2024-11-19T11:14:35,244 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-19T11:14:35,244 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. because compaction request was cancelled 2024-11-19T11:14:35,244 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:C 2024-11-19T11:14:35,244 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-19T11:14:35,244 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 74f9c7f25a644f5292832bdcda4b16f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732014873318 2024-11-19T11:14:35,245 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-19T11:14:35,245 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8eb46aed68cf436cb8160338ee445045, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732014874599 2024-11-19T11:14:35,245 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-19T11:14:35,245 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. because compaction request was cancelled 2024-11-19T11:14:35,245 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:B 2024-11-19T11:14:35,246 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 4327adf3d11c451cb6df6b1bd842d433, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732014874727 2024-11-19T11:14:35,260 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#C#compaction#91 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:35,261 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/f140d207be6e4a05be2042349831ab3a is 50, key is test_row_0/C:col10/1732014874727/Put/seqid=0 2024-11-19T11:14:35,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741928_1104 (size=13289) 2024-11-19T11:14:35,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T11:14:35,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:35,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:35,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:35,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:35,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:35,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 328 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014935447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:35,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:35,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:35,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:35,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/31d5c2f94685411eadb2f03693aa2eba 2024-11-19T11:14:35,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/bc10b1fb9d6c43f8945a8ef952694829 is 50, key is test_row_0/C:col10/1732014875018/Put/seqid=0 2024-11-19T11:14:35,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741929_1105 (size=12301) 2024-11-19T11:14:35,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T11:14:35,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:35,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:35,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:35,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:35,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:35,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T11:14:35,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:35,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:35,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:35,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:35,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:35,681 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/f140d207be6e4a05be2042349831ab3a as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/f140d207be6e4a05be2042349831ab3a 2024-11-19T11:14:35,690 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/C of 9cbe0c20892be91e1583baaaf6ed665f into f140d207be6e4a05be2042349831ab3a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:35,690 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:35,690 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/C, priority=12, startTime=1732014874991; duration=0sec 2024-11-19T11:14:35,690 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:35,690 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:C 2024-11-19T11:14:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T11:14:35,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 329 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014935757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:35,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:35,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:35,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:35,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T11:14:35,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:35,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:35,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:35,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:35,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:35,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/bc10b1fb9d6c43f8945a8ef952694829 2024-11-19T11:14:35,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/c39744ba814749758d74727ae57427d7 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/c39744ba814749758d74727ae57427d7 2024-11-19T11:14:35,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/c39744ba814749758d74727ae57427d7, entries=150, sequenceid=427, filesize=12.0 K 2024-11-19T11:14:35,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/31d5c2f94685411eadb2f03693aa2eba as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/31d5c2f94685411eadb2f03693aa2eba 2024-11-19T11:14:35,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/31d5c2f94685411eadb2f03693aa2eba, entries=150, sequenceid=427, filesize=12.0 K 2024-11-19T11:14:35,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/bc10b1fb9d6c43f8945a8ef952694829 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/bc10b1fb9d6c43f8945a8ef952694829 2024-11-19T11:14:35,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/bc10b1fb9d6c43f8945a8ef952694829, entries=150, sequenceid=427, filesize=12.0 K 2024-11-19T11:14:35,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 9cbe0c20892be91e1583baaaf6ed665f in 880ms, sequenceid=427, compaction requested=true 2024-11-19T11:14:35,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:35,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:14:35,901 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:35,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:35,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:14:35,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:35,901 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:35,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:14:35,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:35,902 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:35,902 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:35,902 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/B is initiating minor compaction (all files) 2024-11-19T11:14:35,902 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/A is initiating minor compaction (all files) 2024-11-19T11:14:35,902 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/B in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:35,902 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/A in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:35,903 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/d92363304f62496b9f5ce96ee9fb34dc, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/4e2e2fe12ac64df787198a9f1afb1761, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/31d5c2f94685411eadb2f03693aa2eba] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=37.0 K 2024-11-19T11:14:35,903 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/3283ecdf1a184e4399899bdcf00c5e8e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/b24f93c994b243f5a9996fc21ce633f3, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/c39744ba814749758d74727ae57427d7] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=39.4 K 2024-11-19T11:14:35,903 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting d92363304f62496b9f5ce96ee9fb34dc, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732014874599 2024-11-19T11:14:35,903 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3283ecdf1a184e4399899bdcf00c5e8e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732014874599 2024-11-19T11:14:35,903 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e2e2fe12ac64df787198a9f1afb1761, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732014874727 2024-11-19T11:14:35,904 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting b24f93c994b243f5a9996fc21ce633f3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732014874727 2024-11-19T11:14:35,904 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 31d5c2f94685411eadb2f03693aa2eba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732014874886 2024-11-19T11:14:35,904 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting c39744ba814749758d74727ae57427d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732014874886 2024-11-19T11:14:35,916 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#B#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:35,917 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/9778e68fb3364876a3b1876045c50222 is 50, key is test_row_0/B:col10/1732014875018/Put/seqid=0 2024-11-19T11:14:35,919 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#A#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:35,920 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/a3be17da1c7e4036b29d558170d24282 is 50, key is test_row_0/A:col10/1732014875018/Put/seqid=0 2024-11-19T11:14:35,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741930_1106 (size=13357) 2024-11-19T11:14:35,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741931_1107 (size=13357) 2024-11-19T11:14:35,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T11:14:35,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:35,951 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T11:14:35,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:35,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:35,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:35,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:35,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:35,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:35,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/543d04cc65d841cf9b7ae98acb824eac is 50, key is test_row_0/A:col10/1732014875116/Put/seqid=0 2024-11-19T11:14:35,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741932_1108 (size=12301) 2024-11-19T11:14:36,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:36,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:36,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:36,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 340 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014936308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:36,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:36,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:36,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:36,355 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/a3be17da1c7e4036b29d558170d24282 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/a3be17da1c7e4036b29d558170d24282 2024-11-19T11:14:36,356 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/9778e68fb3364876a3b1876045c50222 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/9778e68fb3364876a3b1876045c50222 2024-11-19T11:14:36,362 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/B of 9cbe0c20892be91e1583baaaf6ed665f into 9778e68fb3364876a3b1876045c50222(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:36,362 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/A of 9cbe0c20892be91e1583baaaf6ed665f into a3be17da1c7e4036b29d558170d24282(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:36,362 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:36,362 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:36,362 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/A, priority=13, startTime=1732014875901; duration=0sec 2024-11-19T11:14:36,362 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/B, priority=13, startTime=1732014875901; duration=0sec 2024-11-19T11:14:36,362 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:36,362 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:B 2024-11-19T11:14:36,362 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:36,362 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:A 2024-11-19T11:14:36,363 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-19T11:14:36,363 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/543d04cc65d841cf9b7ae98acb824eac 2024-11-19T11:14:36,365 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-19T11:14:36,365 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-19T11:14:36,365 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. because compaction request was cancelled 2024-11-19T11:14:36,365 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:C 2024-11-19T11:14:36,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/b552858b97134a6ab6487007ebad73de is 50, key is test_row_0/B:col10/1732014875116/Put/seqid=0 2024-11-19T11:14:36,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741933_1109 (size=12301) 2024-11-19T11:14:36,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 341 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014936428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:36,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:36,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:36,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:36,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 342 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014936638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:36,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T11:14:36,782 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/b552858b97134a6ab6487007ebad73de 2024-11-19T11:14:36,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/67ca1f6d600442bf974c950a495f3c53 is 50, key is test_row_0/C:col10/1732014875116/Put/seqid=0 2024-11-19T11:14:36,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741934_1110 (size=12301) 2024-11-19T11:14:36,796 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/67ca1f6d600442bf974c950a495f3c53 2024-11-19T11:14:36,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/543d04cc65d841cf9b7ae98acb824eac as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/543d04cc65d841cf9b7ae98acb824eac 2024-11-19T11:14:36,807 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/543d04cc65d841cf9b7ae98acb824eac, entries=150, sequenceid=449, filesize=12.0 K 2024-11-19T11:14:36,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/b552858b97134a6ab6487007ebad73de as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/b552858b97134a6ab6487007ebad73de 2024-11-19T11:14:36,813 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/b552858b97134a6ab6487007ebad73de, entries=150, sequenceid=449, filesize=12.0 K 2024-11-19T11:14:36,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/67ca1f6d600442bf974c950a495f3c53 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/67ca1f6d600442bf974c950a495f3c53 2024-11-19T11:14:36,819 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/67ca1f6d600442bf974c950a495f3c53, entries=150, sequenceid=449, filesize=12.0 K 2024-11-19T11:14:36,820 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 9cbe0c20892be91e1583baaaf6ed665f in 869ms, sequenceid=449, compaction requested=true 2024-11-19T11:14:36,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:36,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:36,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-19T11:14:36,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-19T11:14:36,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-19T11:14:36,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2570 sec 2024-11-19T11:14:36,826 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.2750 sec 2024-11-19T11:14:36,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:36,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T11:14:36,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:36,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:36,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:36,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:36,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:36,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:36,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/8313fb673f754982be47fde228a9d1fd is 50, key is test_row_0/A:col10/1732014876948/Put/seqid=0 2024-11-19T11:14:36,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741935_1111 (size=14741) 2024-11-19T11:14:37,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:37,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 359 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014937009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:37,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 360 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014937118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:37,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:37,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:37,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:37,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 361 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014937327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:37,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:37,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:37,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:37,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/8313fb673f754982be47fde228a9d1fd 2024-11-19T11:14:37,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/7415a609bf874ccd9ec5cd9f4625ff6e is 50, key is test_row_0/B:col10/1732014876948/Put/seqid=0 2024-11-19T11:14:37,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741936_1112 (size=12301) 2024-11-19T11:14:37,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:37,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 362 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014937638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:37,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:37,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:37,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:37,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/7415a609bf874ccd9ec5cd9f4625ff6e 2024-11-19T11:14:37,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/8d32eecb472347fda7fa25d8519a8abf is 50, key is test_row_0/C:col10/1732014876948/Put/seqid=0 2024-11-19T11:14:37,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741937_1113 (size=12301) 2024-11-19T11:14:38,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 363 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014938147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:38,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:38,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:38,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:38,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/8d32eecb472347fda7fa25d8519a8abf 2024-11-19T11:14:38,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/8313fb673f754982be47fde228a9d1fd as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/8313fb673f754982be47fde228a9d1fd 2024-11-19T11:14:38,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/8313fb673f754982be47fde228a9d1fd, entries=200, sequenceid=465, filesize=14.4 K 2024-11-19T11:14:38,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/7415a609bf874ccd9ec5cd9f4625ff6e as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/7415a609bf874ccd9ec5cd9f4625ff6e 2024-11-19T11:14:38,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/7415a609bf874ccd9ec5cd9f4625ff6e, entries=150, sequenceid=465, filesize=12.0 K 2024-11-19T11:14:38,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/8d32eecb472347fda7fa25d8519a8abf as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/8d32eecb472347fda7fa25d8519a8abf 2024-11-19T11:14:38,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/8d32eecb472347fda7fa25d8519a8abf, entries=150, sequenceid=465, filesize=12.0 K 2024-11-19T11:14:38,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 9cbe0c20892be91e1583baaaf6ed665f in 1301ms, sequenceid=465, compaction requested=true 2024-11-19T11:14:38,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:38,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:14:38,250 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:38,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:38,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:14:38,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:38,251 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:38,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:14:38,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:38,252 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:38,252 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:38,252 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/B is initiating minor compaction (all files) 2024-11-19T11:14:38,252 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/A is initiating minor compaction (all files) 2024-11-19T11:14:38,252 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/B in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:38,252 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/A in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:38,252 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/9778e68fb3364876a3b1876045c50222, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/b552858b97134a6ab6487007ebad73de, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/7415a609bf874ccd9ec5cd9f4625ff6e] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=37.1 K 2024-11-19T11:14:38,252 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/a3be17da1c7e4036b29d558170d24282, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/543d04cc65d841cf9b7ae98acb824eac, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/8313fb673f754982be47fde228a9d1fd] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=39.5 K 2024-11-19T11:14:38,253 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3be17da1c7e4036b29d558170d24282, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732014874886 2024-11-19T11:14:38,253 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 9778e68fb3364876a3b1876045c50222, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732014874886 2024-11-19T11:14:38,253 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting b552858b97134a6ab6487007ebad73de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732014875051 2024-11-19T11:14:38,253 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 543d04cc65d841cf9b7ae98acb824eac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732014875051 2024-11-19T11:14:38,254 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 7415a609bf874ccd9ec5cd9f4625ff6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732014876272 2024-11-19T11:14:38,255 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8313fb673f754982be47fde228a9d1fd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732014876272 2024-11-19T11:14:38,274 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#B#compaction#101 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:38,275 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/32e1e906543047b4bd1525635ddcab95 is 50, key is test_row_0/B:col10/1732014876948/Put/seqid=0 2024-11-19T11:14:38,286 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#A#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:38,287 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/ea67d31e35c94fa99884d374e248bbfc is 50, key is test_row_0/A:col10/1732014876948/Put/seqid=0 2024-11-19T11:14:38,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741938_1114 (size=13459) 2024-11-19T11:14:38,327 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/32e1e906543047b4bd1525635ddcab95 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/32e1e906543047b4bd1525635ddcab95 2024-11-19T11:14:38,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741939_1115 (size=13459) 2024-11-19T11:14:38,335 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/B of 9cbe0c20892be91e1583baaaf6ed665f into 32e1e906543047b4bd1525635ddcab95(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:38,335 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:38,335 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/B, priority=13, startTime=1732014878251; duration=0sec 2024-11-19T11:14:38,335 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:38,335 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:B 2024-11-19T11:14:38,335 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:14:38,337 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:14:38,337 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/C is initiating minor compaction (all files) 2024-11-19T11:14:38,337 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/C in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:38,337 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/f140d207be6e4a05be2042349831ab3a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/bc10b1fb9d6c43f8945a8ef952694829, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/67ca1f6d600442bf974c950a495f3c53, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/8d32eecb472347fda7fa25d8519a8abf] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=49.0 K 2024-11-19T11:14:38,338 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting f140d207be6e4a05be2042349831ab3a, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732014874727 2024-11-19T11:14:38,339 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting bc10b1fb9d6c43f8945a8ef952694829, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732014874886 2024-11-19T11:14:38,339 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 67ca1f6d600442bf974c950a495f3c53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732014875051 2024-11-19T11:14:38,340 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d32eecb472347fda7fa25d8519a8abf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732014876272 2024-11-19T11:14:38,353 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#C#compaction#103 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:38,356 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/76b2e62b6854450cabf9fb66dfafaae0 is 50, key is test_row_0/C:col10/1732014876948/Put/seqid=0 2024-11-19T11:14:38,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741940_1116 (size=13425) 2024-11-19T11:14:38,383 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/76b2e62b6854450cabf9fb66dfafaae0 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/76b2e62b6854450cabf9fb66dfafaae0 2024-11-19T11:14:38,390 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/C of 9cbe0c20892be91e1583baaaf6ed665f into 76b2e62b6854450cabf9fb66dfafaae0(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:38,390 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:38,390 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/C, priority=12, startTime=1732014878251; duration=0sec 2024-11-19T11:14:38,391 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:38,391 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:C 2024-11-19T11:14:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T11:14:38,718 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:14:38,718 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:14:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-19T11:14:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T11:14:38,720 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:14:38,721 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:14:38,721 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:14:38,735 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/ea67d31e35c94fa99884d374e248bbfc as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/ea67d31e35c94fa99884d374e248bbfc 2024-11-19T11:14:38,740 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/A of 9cbe0c20892be91e1583baaaf6ed665f into ea67d31e35c94fa99884d374e248bbfc(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:38,740 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:38,740 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/A, priority=13, startTime=1732014878250; duration=0sec 2024-11-19T11:14:38,740 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:38,740 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:A 2024-11-19T11:14:38,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T11:14:38,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T11:14:38,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:38,873 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T11:14:38,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:38,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:38,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:38,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:38,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:38,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:38,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/77affbb8748149e8ba4690bd3302eac4 is 50, key is test_row_0/A:col10/1732014876984/Put/seqid=0 2024-11-19T11:14:38,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741941_1117 (size=12301) 2024-11-19T11:14:39,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T11:14:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:39,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. as already flushing 2024-11-19T11:14:39,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:39,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 375 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014939254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:39,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:39,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:39,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:39,284 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/77affbb8748149e8ba4690bd3302eac4 2024-11-19T11:14:39,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/caed20161ccb4697ae1cad6dac96aa18 is 50, key is test_row_0/B:col10/1732014876984/Put/seqid=0 2024-11-19T11:14:39,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741942_1118 (size=12301) 2024-11-19T11:14:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T11:14:39,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 376 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014939367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:39,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:39,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 377 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014939578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:39,708 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/caed20161ccb4697ae1cad6dac96aa18 2024-11-19T11:14:39,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/e760b9f56b0449eda2f92633ffb534da is 50, key is test_row_0/C:col10/1732014876984/Put/seqid=0 2024-11-19T11:14:39,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741943_1119 (size=12301) 2024-11-19T11:14:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T11:14:39,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 378 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46212 deadline: 1732014939888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:39,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:39,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:39,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:14:40,121 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/e760b9f56b0449eda2f92633ffb534da 2024-11-19T11:14:40,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/77affbb8748149e8ba4690bd3302eac4 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/77affbb8748149e8ba4690bd3302eac4 2024-11-19T11:14:40,132 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/77affbb8748149e8ba4690bd3302eac4, entries=150, sequenceid=486, filesize=12.0 K 2024-11-19T11:14:40,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/caed20161ccb4697ae1cad6dac96aa18 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/caed20161ccb4697ae1cad6dac96aa18 2024-11-19T11:14:40,138 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/caed20161ccb4697ae1cad6dac96aa18, entries=150, sequenceid=486, filesize=12.0 K 2024-11-19T11:14:40,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/e760b9f56b0449eda2f92633ffb534da as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/e760b9f56b0449eda2f92633ffb534da 2024-11-19T11:14:40,145 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/e760b9f56b0449eda2f92633ffb534da, entries=150, sequenceid=486, filesize=12.0 K 2024-11-19T11:14:40,146 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 9cbe0c20892be91e1583baaaf6ed665f in 1273ms, sequenceid=486, compaction requested=false 2024-11-19T11:14:40,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:40,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:40,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-19T11:14:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-19T11:14:40,149 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-19T11:14:40,149 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4260 sec 2024-11-19T11:14:40,151 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.4320 sec 2024-11-19T11:14:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:40,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T11:14:40,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:40,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:40,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:40,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:40,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:40,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:40,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/e69852fdab9d47ef9c97e72683fcee69 is 50, key is test_row_0/A:col10/1732014880398/Put/seqid=0 2024-11-19T11:14:40,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741944_1120 (size=12301) 2024-11-19T11:14:40,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/e69852fdab9d47ef9c97e72683fcee69 2024-11-19T11:14:40,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/0910f44883024d4c95ba901530198a3d is 50, key is test_row_0/B:col10/1732014880398/Put/seqid=0 2024-11-19T11:14:40,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741945_1121 (size=12301) 2024-11-19T11:14:40,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/0910f44883024d4c95ba901530198a3d 2024-11-19T11:14:40,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/f65fa64f29aa426992cc6a724033c85e is 50, key is test_row_0/C:col10/1732014880398/Put/seqid=0 2024-11-19T11:14:40,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741946_1122 (size=12301) 2024-11-19T11:14:40,477 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-19T11:14:40,477 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-19T11:14:40,477 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:14:40,477 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:14:40,481 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-19T11:14:40,481 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:14:40,483 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:40,483 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:40,483 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:40,483 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:40,483 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:40,483 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:14:40,483 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:14:40,484 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:14:40,485 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:40,486 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-19T11:14:40,486 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:14:40,486 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:40,486 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:40,486 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:14:40,486 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-19T11:14:40,486 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:14:40,486 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:40,486 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:40,486 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:14:40,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/f65fa64f29aa426992cc6a724033c85e 2024-11-19T11:14:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T11:14:40,868 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:14:40,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/e69852fdab9d47ef9c97e72683fcee69 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/e69852fdab9d47ef9c97e72683fcee69 2024-11-19T11:14:40,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/e69852fdab9d47ef9c97e72683fcee69, entries=150, sequenceid=501, filesize=12.0 K 2024-11-19T11:14:40,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/0910f44883024d4c95ba901530198a3d as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/0910f44883024d4c95ba901530198a3d 2024-11-19T11:14:40,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/0910f44883024d4c95ba901530198a3d, entries=150, sequenceid=501, filesize=12.0 K 2024-11-19T11:14:40,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/f65fa64f29aa426992cc6a724033c85e as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/f65fa64f29aa426992cc6a724033c85e 2024-11-19T11:14:40,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/f65fa64f29aa426992cc6a724033c85e, entries=150, sequenceid=501, filesize=12.0 K 2024-11-19T11:14:40,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 9cbe0c20892be91e1583baaaf6ed665f in 486ms, sequenceid=501, compaction requested=true 2024-11-19T11:14:40,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:40,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:14:40,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:40,885 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:40,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:14:40,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:40,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbe0c20892be91e1583baaaf6ed665f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:14:40,885 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:40,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:40,886 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:40,886 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:40,886 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/A is initiating minor compaction (all files) 2024-11-19T11:14:40,886 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/B is initiating minor compaction (all files) 2024-11-19T11:14:40,886 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/A in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:40,886 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/B in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:40,886 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/ea67d31e35c94fa99884d374e248bbfc, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/77affbb8748149e8ba4690bd3302eac4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/e69852fdab9d47ef9c97e72683fcee69] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=37.2 K 2024-11-19T11:14:40,886 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/32e1e906543047b4bd1525635ddcab95, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/caed20161ccb4697ae1cad6dac96aa18, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/0910f44883024d4c95ba901530198a3d] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=37.2 K 2024-11-19T11:14:40,887 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea67d31e35c94fa99884d374e248bbfc, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732014876272 2024-11-19T11:14:40,887 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 32e1e906543047b4bd1525635ddcab95, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732014876272 2024-11-19T11:14:40,887 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77affbb8748149e8ba4690bd3302eac4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1732014876984 2024-11-19T11:14:40,887 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting caed20161ccb4697ae1cad6dac96aa18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1732014876984 2024-11-19T11:14:40,887 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting e69852fdab9d47ef9c97e72683fcee69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1732014879229 2024-11-19T11:14:40,887 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 0910f44883024d4c95ba901530198a3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1732014879229 2024-11-19T11:14:40,894 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#A#compaction#110 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:40,895 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/ffd1c9490d6c4d92b73afa4e68590f88 is 50, key is test_row_0/A:col10/1732014880398/Put/seqid=0 2024-11-19T11:14:40,895 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#B#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:40,896 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/070555dcdc0044b0856eed3d78de417c is 50, key is test_row_0/B:col10/1732014880398/Put/seqid=0 2024-11-19T11:14:40,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741947_1123 (size=13561) 2024-11-19T11:14:40,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741948_1124 (size=13561) 2024-11-19T11:14:41,304 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/ffd1c9490d6c4d92b73afa4e68590f88 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/ffd1c9490d6c4d92b73afa4e68590f88 2024-11-19T11:14:41,304 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/070555dcdc0044b0856eed3d78de417c as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/070555dcdc0044b0856eed3d78de417c 2024-11-19T11:14:41,309 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/B of 9cbe0c20892be91e1583baaaf6ed665f into 070555dcdc0044b0856eed3d78de417c(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:41,309 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/A of 9cbe0c20892be91e1583baaaf6ed665f into ffd1c9490d6c4d92b73afa4e68590f88(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:41,309 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:41,309 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:41,309 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/B, priority=13, startTime=1732014880885; duration=0sec 2024-11-19T11:14:41,309 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/A, priority=13, startTime=1732014880885; duration=0sec 2024-11-19T11:14:41,309 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:41,310 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:41,310 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:A 2024-11-19T11:14:41,310 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:B 2024-11-19T11:14:41,310 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:41,311 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:41,311 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 9cbe0c20892be91e1583baaaf6ed665f/C is initiating minor compaction (all files) 2024-11-19T11:14:41,311 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbe0c20892be91e1583baaaf6ed665f/C in TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:41,311 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/76b2e62b6854450cabf9fb66dfafaae0, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/e760b9f56b0449eda2f92633ffb534da, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/f65fa64f29aa426992cc6a724033c85e] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp, totalSize=37.1 K 2024-11-19T11:14:41,311 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76b2e62b6854450cabf9fb66dfafaae0, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732014876272 2024-11-19T11:14:41,311 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting e760b9f56b0449eda2f92633ffb534da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1732014876984 2024-11-19T11:14:41,312 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting f65fa64f29aa426992cc6a724033c85e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1732014879229 2024-11-19T11:14:41,320 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbe0c20892be91e1583baaaf6ed665f#C#compaction#112 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:41,320 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/2997b3cf27a44de4aa12d969a405c15e is 50, key is test_row_0/C:col10/1732014880398/Put/seqid=0 2024-11-19T11:14:41,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741949_1125 (size=13527) 2024-11-19T11:14:41,736 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/2997b3cf27a44de4aa12d969a405c15e as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/2997b3cf27a44de4aa12d969a405c15e 2024-11-19T11:14:41,742 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbe0c20892be91e1583baaaf6ed665f/C of 9cbe0c20892be91e1583baaaf6ed665f into 2997b3cf27a44de4aa12d969a405c15e(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:41,742 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:41,742 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., storeName=9cbe0c20892be91e1583baaaf6ed665f/C, priority=13, startTime=1732014880885; duration=0sec 2024-11-19T11:14:41,742 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:41,742 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbe0c20892be91e1583baaaf6ed665f:C 2024-11-19T11:14:42,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:42,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T11:14:42,161 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-19T11:14:42,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:42,161 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:14:42,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:42,161 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:42,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:42,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:42,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:42,161 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:42,161 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:14:42,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:42,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/dc270780d3f948b9b58c06333e5583e3 is 50, key is test_row_0/A:col10/1732014880453/Put/seqid=0 2024-11-19T11:14:42,169 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-19T11:14:42,169 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:14:42,169 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:42,169 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:42,169 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:14:42,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741950_1126 (size=12301) 2024-11-19T11:14:42,190 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-19T11:14:42,190 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:14:42,190 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:42,190 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:42,190 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:14:42,199 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-19T11:14:42,199 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:14:42,199 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:42,199 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:42,199 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:14:42,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T11:14:42,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-19T11:14:42,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-19T11:14:42,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-19T11:14:42,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-19T11:14:42,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 329 2024-11-19T11:14:42,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T11:14:42,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3143 2024-11-19T11:14:42,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2946 2024-11-19T11:14:42,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T11:14:42,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1325 2024-11-19T11:14:42,201 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3971 rows 2024-11-19T11:14:42,201 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1323 2024-11-19T11:14:42,201 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3962 rows 2024-11-19T11:14:42,201 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T11:14:42,201 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T11:14:42,201 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:42,202 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:42,202 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:14:42,209 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T11:14:42,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T11:14:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T11:14:42,219 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014882219"}]},"ts":"1732014882219"} 2024-11-19T11:14:42,221 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T11:14:42,221 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T11:14:42,223 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T11:14:42,227 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbe0c20892be91e1583baaaf6ed665f, UNASSIGN}] 2024-11-19T11:14:42,228 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbe0c20892be91e1583baaaf6ed665f, UNASSIGN 2024-11-19T11:14:42,229 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=9cbe0c20892be91e1583baaaf6ed665f, regionState=CLOSING, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:14:42,231 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbe0c20892be91e1583baaaf6ed665f, UNASSIGN because future has completed 2024-11-19T11:14:42,232 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T11:14:42,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640}] 2024-11-19T11:14:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T11:14:42,392 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:42,392 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T11:14:42,394 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 9cbe0c20892be91e1583baaaf6ed665f, disabling compactions & flushes 2024-11-19T11:14:42,395 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:42,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T11:14:42,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/dc270780d3f948b9b58c06333e5583e3 2024-11-19T11:14:42,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/8f512ba005884634b37ea893c7d2166d is 50, key is test_row_0/B:col10/1732014880453/Put/seqid=0 2024-11-19T11:14:42,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741951_1127 (size=12301) 2024-11-19T11:14:42,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T11:14:42,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/8f512ba005884634b37ea893c7d2166d 2024-11-19T11:14:43,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/2f3c5964200f49bb83eaa4ea754ebd22 is 50, key is test_row_0/C:col10/1732014880453/Put/seqid=0 2024-11-19T11:14:43,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741952_1128 (size=12301) 2024-11-19T11:14:43,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T11:14:43,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/2f3c5964200f49bb83eaa4ea754ebd22 2024-11-19T11:14:43,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/dc270780d3f948b9b58c06333e5583e3 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/dc270780d3f948b9b58c06333e5583e3 2024-11-19T11:14:43,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/dc270780d3f948b9b58c06333e5583e3, entries=150, sequenceid=522, filesize=12.0 K 2024-11-19T11:14:43,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/8f512ba005884634b37ea893c7d2166d as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/8f512ba005884634b37ea893c7d2166d 2024-11-19T11:14:43,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/8f512ba005884634b37ea893c7d2166d, entries=150, sequenceid=522, filesize=12.0 K 2024-11-19T11:14:43,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/2f3c5964200f49bb83eaa4ea754ebd22 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/2f3c5964200f49bb83eaa4ea754ebd22 2024-11-19T11:14:43,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/2f3c5964200f49bb83eaa4ea754ebd22, entries=150, sequenceid=522, filesize=12.0 K 2024-11-19T11:14:43,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=20.13 KB/20610 for 9cbe0c20892be91e1583baaaf6ed665f in 1278ms, sequenceid=522, compaction requested=false 2024-11-19T11:14:43,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbe0c20892be91e1583baaaf6ed665f: 2024-11-19T11:14:43,438 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:43,438 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:43,439 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. after waiting 0 ms 2024-11-19T11:14:43,439 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:43,439 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 9cbe0c20892be91e1583baaaf6ed665f 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-19T11:14:43,439 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=A 2024-11-19T11:14:43,439 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:43,439 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=B 2024-11-19T11:14:43,439 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:43,439 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9cbe0c20892be91e1583baaaf6ed665f, store=C 2024-11-19T11:14:43,439 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:43,443 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/f71f13294bd142209de3b210c877f882 is 50, key is test_row_1/A:col10/1732014882197/Put/seqid=0 2024-11-19T11:14:43,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741953_1129 (size=9857) 2024-11-19T11:14:43,566 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T11:14:43,847 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/f71f13294bd142209de3b210c877f882 2024-11-19T11:14:43,855 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/7e0c695fbd9344c98940a4769130e6eb is 50, key is test_row_1/B:col10/1732014882197/Put/seqid=0 2024-11-19T11:14:43,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741954_1130 (size=9857) 2024-11-19T11:14:44,261 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/7e0c695fbd9344c98940a4769130e6eb 2024-11-19T11:14:44,322 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/6f793671b6874fc0834804c14d9d6487 is 50, key is test_row_1/C:col10/1732014882197/Put/seqid=0 2024-11-19T11:14:44,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741955_1131 (size=9857) 2024-11-19T11:14:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T11:14:44,728 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/6f793671b6874fc0834804c14d9d6487 2024-11-19T11:14:44,733 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/A/f71f13294bd142209de3b210c877f882 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/f71f13294bd142209de3b210c877f882 2024-11-19T11:14:44,738 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/f71f13294bd142209de3b210c877f882, entries=100, sequenceid=528, filesize=9.6 K 2024-11-19T11:14:44,739 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/B/7e0c695fbd9344c98940a4769130e6eb as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/7e0c695fbd9344c98940a4769130e6eb 2024-11-19T11:14:44,744 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/7e0c695fbd9344c98940a4769130e6eb, entries=100, sequenceid=528, filesize=9.6 K 2024-11-19T11:14:44,745 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/.tmp/C/6f793671b6874fc0834804c14d9d6487 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/6f793671b6874fc0834804c14d9d6487 2024-11-19T11:14:44,749 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/6f793671b6874fc0834804c14d9d6487, entries=100, sequenceid=528, filesize=9.6 K 2024-11-19T11:14:44,750 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 9cbe0c20892be91e1583baaaf6ed665f in 1311ms, sequenceid=528, compaction requested=true 2024-11-19T11:14:44,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/2d652fdbd9ed44e8bc55b69324403caf, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/682193c053594e4583687cf17401f0b0, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/d2e69394289f44b08dcc6e753b3db538, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/d5970b0d670a499086188ea7611e032f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/7d3351937a794cd3b66765557b56a24f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/340057bdf5ca43e6b2ac9d93205e19b7, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/479f4110df1c450d89df820f6e747ded, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/8086b01099c54b26bda3b74bcb8f5614, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/fa710919571247ac91ecedecf8297d83, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/eb405d91f113406a8802392532074764, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/e16ec392e0d64a96adb162d328243069, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/c239eb61378645f0a3219059d57c3ce1, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/6400a5d6fd5d4209911c99c56a691152, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/c04c2fd9254546e1b8f9c71f7d871594, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/3d2d7a729e2f4675b85a5bc62513ae54, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/0f7e5c1c1c8d471db114c38b1dae6007, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/9db9c472f5414768bcc1b5249f9a3f8b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/78422ee05be642fe95331c5c45e1f701, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/529b71a91142424cbd2e57d09862e342, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/55aa0f703bcd4b0b83ec5e82d2dd5581, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/750553ba3e4047b4acfa89e3f2b5c199, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/86541d5799414f6d8be8c6a70c1eb3e0, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/ccdaec296a54436cbe5d35630296022a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/4051293baa7447d7b33537aaebe3a5db, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/4357a63cf66849fd8b90cc4e55890e7d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/3f009cffdff3457981c04c229818a25e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/46c8d4b715214564ac19b7444cfa0e6e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/a31fdb9334b64f888a71df8ad6ff13ff, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/3283ecdf1a184e4399899bdcf00c5e8e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/b24f93c994b243f5a9996fc21ce633f3, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/a3be17da1c7e4036b29d558170d24282, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/c39744ba814749758d74727ae57427d7, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/543d04cc65d841cf9b7ae98acb824eac, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/8313fb673f754982be47fde228a9d1fd, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/ea67d31e35c94fa99884d374e248bbfc, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/77affbb8748149e8ba4690bd3302eac4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/e69852fdab9d47ef9c97e72683fcee69] to archive 2024-11-19T11:14:44,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T11:14:44,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/2d652fdbd9ed44e8bc55b69324403caf to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/2d652fdbd9ed44e8bc55b69324403caf 2024-11-19T11:14:44,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/682193c053594e4583687cf17401f0b0 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/682193c053594e4583687cf17401f0b0 2024-11-19T11:14:44,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/d2e69394289f44b08dcc6e753b3db538 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/d2e69394289f44b08dcc6e753b3db538 2024-11-19T11:14:44,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/d5970b0d670a499086188ea7611e032f to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/d5970b0d670a499086188ea7611e032f 2024-11-19T11:14:44,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/7d3351937a794cd3b66765557b56a24f to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/7d3351937a794cd3b66765557b56a24f 2024-11-19T11:14:44,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/340057bdf5ca43e6b2ac9d93205e19b7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/340057bdf5ca43e6b2ac9d93205e19b7 2024-11-19T11:14:44,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/479f4110df1c450d89df820f6e747ded to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/479f4110df1c450d89df820f6e747ded 2024-11-19T11:14:44,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/8086b01099c54b26bda3b74bcb8f5614 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/8086b01099c54b26bda3b74bcb8f5614 2024-11-19T11:14:44,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/fa710919571247ac91ecedecf8297d83 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/fa710919571247ac91ecedecf8297d83 2024-11-19T11:14:44,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/eb405d91f113406a8802392532074764 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/eb405d91f113406a8802392532074764 2024-11-19T11:14:44,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/e16ec392e0d64a96adb162d328243069 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/e16ec392e0d64a96adb162d328243069 2024-11-19T11:14:44,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/c239eb61378645f0a3219059d57c3ce1 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/c239eb61378645f0a3219059d57c3ce1 2024-11-19T11:14:44,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/6400a5d6fd5d4209911c99c56a691152 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/6400a5d6fd5d4209911c99c56a691152 2024-11-19T11:14:44,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/c04c2fd9254546e1b8f9c71f7d871594 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/c04c2fd9254546e1b8f9c71f7d871594 2024-11-19T11:14:44,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/3d2d7a729e2f4675b85a5bc62513ae54 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/3d2d7a729e2f4675b85a5bc62513ae54 2024-11-19T11:14:44,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/0f7e5c1c1c8d471db114c38b1dae6007 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/0f7e5c1c1c8d471db114c38b1dae6007 2024-11-19T11:14:44,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/9db9c472f5414768bcc1b5249f9a3f8b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/9db9c472f5414768bcc1b5249f9a3f8b 2024-11-19T11:14:44,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/78422ee05be642fe95331c5c45e1f701 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/78422ee05be642fe95331c5c45e1f701 2024-11-19T11:14:44,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/529b71a91142424cbd2e57d09862e342 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/529b71a91142424cbd2e57d09862e342 2024-11-19T11:14:44,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/55aa0f703bcd4b0b83ec5e82d2dd5581 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/55aa0f703bcd4b0b83ec5e82d2dd5581 2024-11-19T11:14:44,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/750553ba3e4047b4acfa89e3f2b5c199 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/750553ba3e4047b4acfa89e3f2b5c199 2024-11-19T11:14:44,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/86541d5799414f6d8be8c6a70c1eb3e0 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/86541d5799414f6d8be8c6a70c1eb3e0 2024-11-19T11:14:44,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/ccdaec296a54436cbe5d35630296022a to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/ccdaec296a54436cbe5d35630296022a 2024-11-19T11:14:44,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/4051293baa7447d7b33537aaebe3a5db to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/4051293baa7447d7b33537aaebe3a5db 2024-11-19T11:14:44,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/4357a63cf66849fd8b90cc4e55890e7d to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/4357a63cf66849fd8b90cc4e55890e7d 2024-11-19T11:14:44,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/3f009cffdff3457981c04c229818a25e to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/3f009cffdff3457981c04c229818a25e 2024-11-19T11:14:44,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/46c8d4b715214564ac19b7444cfa0e6e to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/46c8d4b715214564ac19b7444cfa0e6e 2024-11-19T11:14:44,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/a31fdb9334b64f888a71df8ad6ff13ff to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/a31fdb9334b64f888a71df8ad6ff13ff 2024-11-19T11:14:44,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/3283ecdf1a184e4399899bdcf00c5e8e to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/3283ecdf1a184e4399899bdcf00c5e8e 2024-11-19T11:14:44,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/b24f93c994b243f5a9996fc21ce633f3 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/b24f93c994b243f5a9996fc21ce633f3 2024-11-19T11:14:44,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/a3be17da1c7e4036b29d558170d24282 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/a3be17da1c7e4036b29d558170d24282 2024-11-19T11:14:44,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/c39744ba814749758d74727ae57427d7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/c39744ba814749758d74727ae57427d7 2024-11-19T11:14:44,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/543d04cc65d841cf9b7ae98acb824eac to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/543d04cc65d841cf9b7ae98acb824eac 2024-11-19T11:14:44,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/8313fb673f754982be47fde228a9d1fd to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/8313fb673f754982be47fde228a9d1fd 2024-11-19T11:14:44,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/ea67d31e35c94fa99884d374e248bbfc to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/ea67d31e35c94fa99884d374e248bbfc 2024-11-19T11:14:44,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/77affbb8748149e8ba4690bd3302eac4 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/77affbb8748149e8ba4690bd3302eac4 2024-11-19T11:14:44,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/e69852fdab9d47ef9c97e72683fcee69 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/e69852fdab9d47ef9c97e72683fcee69 2024-11-19T11:14:44,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/f14c7178ac2848e4a713a5475bf5b401, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/21ef635f5174481abf2cb45bf93e409c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/2ae96b7af8fa44ce9a6a6db06f31c4a3, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/9b34eb4d18694fbe935fad1c5f148aae, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/b52a384872ca413f8680c81e8064656e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/92d182811b7c488497d49006de32bcd8, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/7a3bea00bf1b40198568390ca3675a56, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/c20f7421282a41f6838a333dddaa429f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/fe1bf774582149b7826df2b1eb6e45f3, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/c57a216f5d874aa6a98bfc7342af4ac5, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/f5a743c5865c49edb8941510284d198c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/7fe5a07da1e9437d9d2d8fb50814590c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/89b5e010007d4cc5beec1212eb610097, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/b03b4a3a415442f2a7bd95208a52e831, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/8157f81cc8ff475393a8bbc418af52b1, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/9179c824fb1e4c9597820ec7a5224c66, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/c29569d5775949268ece474c65e64ac4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/d83068855951415a97f403b466e14537, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/4ddefab20c924761a13f938212abf0d1, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/d5489b3911d24479a21ca3bb99d9e8c9, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/8dbd464e62f042a0911ba8bf7d11b627, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/8e94f57afade48b490dc849d729cc199, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/5999f47618124442bee622d407ca465b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/3e12d2353ed645e4ac55697b33a47ea0, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/274925480c8b4e029cdf2046b19c1d63, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/e24e1ac8473e44c295a8082ba575b1cb, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/72cdeec4f42b4f218edd1a46085a19b4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/d92363304f62496b9f5ce96ee9fb34dc, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/29adcd98f58c44bfbc4543a14c171752, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/4e2e2fe12ac64df787198a9f1afb1761, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/9778e68fb3364876a3b1876045c50222, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/31d5c2f94685411eadb2f03693aa2eba, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/b552858b97134a6ab6487007ebad73de, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/32e1e906543047b4bd1525635ddcab95, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/7415a609bf874ccd9ec5cd9f4625ff6e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/caed20161ccb4697ae1cad6dac96aa18, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/0910f44883024d4c95ba901530198a3d] to archive 2024-11-19T11:14:44,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T11:14:44,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/f14c7178ac2848e4a713a5475bf5b401 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/f14c7178ac2848e4a713a5475bf5b401 2024-11-19T11:14:44,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/21ef635f5174481abf2cb45bf93e409c to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/21ef635f5174481abf2cb45bf93e409c 2024-11-19T11:14:44,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/2ae96b7af8fa44ce9a6a6db06f31c4a3 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/2ae96b7af8fa44ce9a6a6db06f31c4a3 2024-11-19T11:14:44,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/9b34eb4d18694fbe935fad1c5f148aae to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/9b34eb4d18694fbe935fad1c5f148aae 2024-11-19T11:14:44,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/b52a384872ca413f8680c81e8064656e to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/b52a384872ca413f8680c81e8064656e 2024-11-19T11:14:44,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/92d182811b7c488497d49006de32bcd8 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/92d182811b7c488497d49006de32bcd8 2024-11-19T11:14:44,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/7a3bea00bf1b40198568390ca3675a56 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/7a3bea00bf1b40198568390ca3675a56 2024-11-19T11:14:44,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/c20f7421282a41f6838a333dddaa429f to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/c20f7421282a41f6838a333dddaa429f 2024-11-19T11:14:44,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/fe1bf774582149b7826df2b1eb6e45f3 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/fe1bf774582149b7826df2b1eb6e45f3 2024-11-19T11:14:44,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/c57a216f5d874aa6a98bfc7342af4ac5 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/c57a216f5d874aa6a98bfc7342af4ac5 2024-11-19T11:14:44,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/f5a743c5865c49edb8941510284d198c to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/f5a743c5865c49edb8941510284d198c 2024-11-19T11:14:44,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/7fe5a07da1e9437d9d2d8fb50814590c to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/7fe5a07da1e9437d9d2d8fb50814590c 2024-11-19T11:14:44,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/89b5e010007d4cc5beec1212eb610097 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/89b5e010007d4cc5beec1212eb610097 2024-11-19T11:14:44,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/b03b4a3a415442f2a7bd95208a52e831 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/b03b4a3a415442f2a7bd95208a52e831 2024-11-19T11:14:44,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/8157f81cc8ff475393a8bbc418af52b1 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/8157f81cc8ff475393a8bbc418af52b1 2024-11-19T11:14:44,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/9179c824fb1e4c9597820ec7a5224c66 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/9179c824fb1e4c9597820ec7a5224c66 2024-11-19T11:14:44,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/c29569d5775949268ece474c65e64ac4 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/c29569d5775949268ece474c65e64ac4 2024-11-19T11:14:44,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/d83068855951415a97f403b466e14537 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/d83068855951415a97f403b466e14537 2024-11-19T11:14:44,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/4ddefab20c924761a13f938212abf0d1 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/4ddefab20c924761a13f938212abf0d1 2024-11-19T11:14:44,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/d5489b3911d24479a21ca3bb99d9e8c9 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/d5489b3911d24479a21ca3bb99d9e8c9 2024-11-19T11:14:44,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/8dbd464e62f042a0911ba8bf7d11b627 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/8dbd464e62f042a0911ba8bf7d11b627 2024-11-19T11:14:44,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/8e94f57afade48b490dc849d729cc199 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/8e94f57afade48b490dc849d729cc199 2024-11-19T11:14:44,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/5999f47618124442bee622d407ca465b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/5999f47618124442bee622d407ca465b 2024-11-19T11:14:44,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/3e12d2353ed645e4ac55697b33a47ea0 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/3e12d2353ed645e4ac55697b33a47ea0 2024-11-19T11:14:44,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/274925480c8b4e029cdf2046b19c1d63 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/274925480c8b4e029cdf2046b19c1d63 2024-11-19T11:14:44,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/e24e1ac8473e44c295a8082ba575b1cb to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/e24e1ac8473e44c295a8082ba575b1cb 2024-11-19T11:14:44,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/72cdeec4f42b4f218edd1a46085a19b4 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/72cdeec4f42b4f218edd1a46085a19b4 2024-11-19T11:14:44,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/d92363304f62496b9f5ce96ee9fb34dc to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/d92363304f62496b9f5ce96ee9fb34dc 2024-11-19T11:14:44,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/29adcd98f58c44bfbc4543a14c171752 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/29adcd98f58c44bfbc4543a14c171752 2024-11-19T11:14:44,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/4e2e2fe12ac64df787198a9f1afb1761 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/4e2e2fe12ac64df787198a9f1afb1761 2024-11-19T11:14:44,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/9778e68fb3364876a3b1876045c50222 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/9778e68fb3364876a3b1876045c50222 2024-11-19T11:14:44,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/31d5c2f94685411eadb2f03693aa2eba to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/31d5c2f94685411eadb2f03693aa2eba 2024-11-19T11:14:44,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/b552858b97134a6ab6487007ebad73de to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/b552858b97134a6ab6487007ebad73de 2024-11-19T11:14:44,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/32e1e906543047b4bd1525635ddcab95 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/32e1e906543047b4bd1525635ddcab95 2024-11-19T11:14:44,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/7415a609bf874ccd9ec5cd9f4625ff6e to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/7415a609bf874ccd9ec5cd9f4625ff6e 2024-11-19T11:14:44,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/caed20161ccb4697ae1cad6dac96aa18 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/caed20161ccb4697ae1cad6dac96aa18 2024-11-19T11:14:44,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/0910f44883024d4c95ba901530198a3d to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/0910f44883024d4c95ba901530198a3d 2024-11-19T11:14:44,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/14ccb44a34344ebaa8bd49cae2a9cd38, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/a41843ff19234afdb2d602a34857f26a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/3a771587a42c4df3a13b23e96a6a3fdc, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/17f7ac7080964e5c8fbd4afad44d4676, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/ad95f1f97efd443b90d8a90d35b4aad8, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/5a1d40f3446044c0813679190886a33b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/7f3fbd4440234864b3d40dbac158c582, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/97d753cdef7844e8982c019d4f8d129f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/f2a0240926ca495bad73bdf7351f0f2d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/7fdcafeddb4c49219f1102a1dadd996b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/99da47e8efbf4695a5bc6b414c70e3bd, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/23b039d7678741889d9fa12ce6f6480c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/0409ac796794492cab1975efee553d6e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/427d5dc3046b4adfaab4c17a9c767802, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/284ac5d06fa248eb8c7f1a9b33df86bd, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/b286eeb799554344974a608a657c3504, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/1a36b94a44574b9c9bf127af9f072389, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/0bbde82016994f3fadf88c28fda94e6e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/899a49a5411c4f7f8bc9931d9b87653f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/bd66ca8564824806ac0a2d0d511bbe33, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/90b635d8f4db4076ac9589c8404c5cdd, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/cba72ceb6b764324bd149f7145f0dffd, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/0e39fb045de848269219f6d3748bf2d1, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/6c9de9fa735742f6a3c589c534516b32, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/a995bac1b1ef4416ad62492fa6b19c36, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/f17ed6d1ac984a9da079f46827e4f1b3, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/74f9c7f25a644f5292832bdcda4b16f2, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/8eb46aed68cf436cb8160338ee445045, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/f140d207be6e4a05be2042349831ab3a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/4327adf3d11c451cb6df6b1bd842d433, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/bc10b1fb9d6c43f8945a8ef952694829, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/67ca1f6d600442bf974c950a495f3c53, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/76b2e62b6854450cabf9fb66dfafaae0, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/8d32eecb472347fda7fa25d8519a8abf, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/e760b9f56b0449eda2f92633ffb534da, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/f65fa64f29aa426992cc6a724033c85e] to archive 2024-11-19T11:14:44,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T11:14:44,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/14ccb44a34344ebaa8bd49cae2a9cd38 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/14ccb44a34344ebaa8bd49cae2a9cd38 2024-11-19T11:14:44,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/a41843ff19234afdb2d602a34857f26a to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/a41843ff19234afdb2d602a34857f26a 2024-11-19T11:14:44,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/3a771587a42c4df3a13b23e96a6a3fdc to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/3a771587a42c4df3a13b23e96a6a3fdc 2024-11-19T11:14:44,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/17f7ac7080964e5c8fbd4afad44d4676 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/17f7ac7080964e5c8fbd4afad44d4676 2024-11-19T11:14:44,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/ad95f1f97efd443b90d8a90d35b4aad8 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/ad95f1f97efd443b90d8a90d35b4aad8 2024-11-19T11:14:44,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/5a1d40f3446044c0813679190886a33b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/5a1d40f3446044c0813679190886a33b 2024-11-19T11:14:44,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/7f3fbd4440234864b3d40dbac158c582 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/7f3fbd4440234864b3d40dbac158c582 2024-11-19T11:14:44,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/97d753cdef7844e8982c019d4f8d129f to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/97d753cdef7844e8982c019d4f8d129f 2024-11-19T11:14:44,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/f2a0240926ca495bad73bdf7351f0f2d to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/f2a0240926ca495bad73bdf7351f0f2d 2024-11-19T11:14:44,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/7fdcafeddb4c49219f1102a1dadd996b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/7fdcafeddb4c49219f1102a1dadd996b 2024-11-19T11:14:44,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/99da47e8efbf4695a5bc6b414c70e3bd to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/99da47e8efbf4695a5bc6b414c70e3bd 2024-11-19T11:14:44,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/23b039d7678741889d9fa12ce6f6480c to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/23b039d7678741889d9fa12ce6f6480c 2024-11-19T11:14:44,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/0409ac796794492cab1975efee553d6e to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/0409ac796794492cab1975efee553d6e 2024-11-19T11:14:44,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/427d5dc3046b4adfaab4c17a9c767802 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/427d5dc3046b4adfaab4c17a9c767802 2024-11-19T11:14:44,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/284ac5d06fa248eb8c7f1a9b33df86bd to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/284ac5d06fa248eb8c7f1a9b33df86bd 2024-11-19T11:14:44,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/b286eeb799554344974a608a657c3504 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/b286eeb799554344974a608a657c3504 2024-11-19T11:14:44,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/1a36b94a44574b9c9bf127af9f072389 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/1a36b94a44574b9c9bf127af9f072389 2024-11-19T11:14:44,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/0bbde82016994f3fadf88c28fda94e6e to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/0bbde82016994f3fadf88c28fda94e6e 2024-11-19T11:14:44,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/899a49a5411c4f7f8bc9931d9b87653f to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/899a49a5411c4f7f8bc9931d9b87653f 2024-11-19T11:14:44,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/bd66ca8564824806ac0a2d0d511bbe33 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/bd66ca8564824806ac0a2d0d511bbe33 2024-11-19T11:14:44,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/90b635d8f4db4076ac9589c8404c5cdd to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/90b635d8f4db4076ac9589c8404c5cdd 2024-11-19T11:14:44,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/cba72ceb6b764324bd149f7145f0dffd to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/cba72ceb6b764324bd149f7145f0dffd 2024-11-19T11:14:44,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/0e39fb045de848269219f6d3748bf2d1 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/0e39fb045de848269219f6d3748bf2d1 2024-11-19T11:14:44,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/6c9de9fa735742f6a3c589c534516b32 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/6c9de9fa735742f6a3c589c534516b32 2024-11-19T11:14:44,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/a995bac1b1ef4416ad62492fa6b19c36 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/a995bac1b1ef4416ad62492fa6b19c36 2024-11-19T11:14:44,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/f17ed6d1ac984a9da079f46827e4f1b3 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/f17ed6d1ac984a9da079f46827e4f1b3 2024-11-19T11:14:44,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/74f9c7f25a644f5292832bdcda4b16f2 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/74f9c7f25a644f5292832bdcda4b16f2 2024-11-19T11:14:44,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/8eb46aed68cf436cb8160338ee445045 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/8eb46aed68cf436cb8160338ee445045 2024-11-19T11:14:44,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/f140d207be6e4a05be2042349831ab3a to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/f140d207be6e4a05be2042349831ab3a 2024-11-19T11:14:44,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/4327adf3d11c451cb6df6b1bd842d433 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/4327adf3d11c451cb6df6b1bd842d433 2024-11-19T11:14:44,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/bc10b1fb9d6c43f8945a8ef952694829 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/bc10b1fb9d6c43f8945a8ef952694829 2024-11-19T11:14:44,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/67ca1f6d600442bf974c950a495f3c53 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/67ca1f6d600442bf974c950a495f3c53 2024-11-19T11:14:44,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/76b2e62b6854450cabf9fb66dfafaae0 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/76b2e62b6854450cabf9fb66dfafaae0 2024-11-19T11:14:44,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/8d32eecb472347fda7fa25d8519a8abf to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/8d32eecb472347fda7fa25d8519a8abf 2024-11-19T11:14:44,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/e760b9f56b0449eda2f92633ffb534da to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/e760b9f56b0449eda2f92633ffb534da 2024-11-19T11:14:44,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/f65fa64f29aa426992cc6a724033c85e to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/f65fa64f29aa426992cc6a724033c85e 2024-11-19T11:14:44,913 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/recovered.edits/531.seqid, newMaxSeqId=531, maxSeqId=1 2024-11-19T11:14:44,914 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. 2024-11-19T11:14:44,914 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 9cbe0c20892be91e1583baaaf6ed665f: Waiting for close lock at 1732014882394Running coprocessor pre-close hooks at 1732014882394Disabling compacts and flushes for region at 1732014882394Disabling writes for close at 1732014883439 (+1045 ms)Obtaining lock to block concurrent updates at 1732014883439Preparing flush snapshotting stores in 9cbe0c20892be91e1583baaaf6ed665f at 1732014883439Finished memstore snapshotting TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732014883439Flushing stores of TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f. at 1732014883440 (+1 ms)Flushing 9cbe0c20892be91e1583baaaf6ed665f/A: creating writer at 1732014883440Flushing 9cbe0c20892be91e1583baaaf6ed665f/A: appending metadata at 1732014883442 (+2 ms)Flushing 9cbe0c20892be91e1583baaaf6ed665f/A: closing flushed file at 1732014883442Flushing 9cbe0c20892be91e1583baaaf6ed665f/B: creating writer at 1732014883852 (+410 ms)Flushing 9cbe0c20892be91e1583baaaf6ed665f/B: appending metadata at 1732014883855 (+3 ms)Flushing 9cbe0c20892be91e1583baaaf6ed665f/B: closing flushed file at 1732014883855Flushing 9cbe0c20892be91e1583baaaf6ed665f/C: creating writer at 1732014884317 (+462 ms)Flushing 9cbe0c20892be91e1583baaaf6ed665f/C: appending metadata at 1732014884322 (+5 ms)Flushing 9cbe0c20892be91e1583baaaf6ed665f/C: closing flushed file at 1732014884322Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a2702a1: reopening flushed file at 1732014884732 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3434eb12: reopening flushed file at 1732014884738 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ee1e735: reopening flushed file at 1732014884744 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 9cbe0c20892be91e1583baaaf6ed665f in 1311ms, sequenceid=528, compaction requested=true at 1732014884750 (+6 ms)Writing region close event to WAL at 1732014884910 (+160 ms)Running coprocessor post-close hooks at 1732014884914 (+4 ms)Closed at 1732014884914 2024-11-19T11:14:44,917 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:44,918 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=9cbe0c20892be91e1583baaaf6ed665f, regionState=CLOSED 2024-11-19T11:14:44,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 because future has completed 2024-11-19T11:14:44,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-19T11:14:44,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 9cbe0c20892be91e1583baaaf6ed665f, server=16c88966c0af,35833,1732014855640 in 2.6890 sec 2024-11-19T11:14:44,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-19T11:14:44,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbe0c20892be91e1583baaaf6ed665f, UNASSIGN in 2.6960 sec 2024-11-19T11:14:44,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-19T11:14:44,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.7020 sec 2024-11-19T11:14:44,929 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014884928"}]},"ts":"1732014884928"} 2024-11-19T11:14:44,930 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T11:14:44,930 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T11:14:44,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.7190 sec 2024-11-19T11:14:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T11:14:46,388 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:14:46,392 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T11:14:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:14:46,399 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:14:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T11:14:46,401 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:14:46,408 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:46,412 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A, FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B, FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C, FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/recovered.edits] 2024-11-19T11:14:46,418 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/dc270780d3f948b9b58c06333e5583e3 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/dc270780d3f948b9b58c06333e5583e3 2024-11-19T11:14:46,420 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/f71f13294bd142209de3b210c877f882 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/f71f13294bd142209de3b210c877f882 2024-11-19T11:14:46,427 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/ffd1c9490d6c4d92b73afa4e68590f88 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/A/ffd1c9490d6c4d92b73afa4e68590f88 2024-11-19T11:14:46,430 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/070555dcdc0044b0856eed3d78de417c to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/070555dcdc0044b0856eed3d78de417c 2024-11-19T11:14:46,432 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/7e0c695fbd9344c98940a4769130e6eb to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/7e0c695fbd9344c98940a4769130e6eb 2024-11-19T11:14:46,434 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/8f512ba005884634b37ea893c7d2166d to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/B/8f512ba005884634b37ea893c7d2166d 2024-11-19T11:14:46,441 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/2997b3cf27a44de4aa12d969a405c15e to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/2997b3cf27a44de4aa12d969a405c15e 2024-11-19T11:14:46,443 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/2f3c5964200f49bb83eaa4ea754ebd22 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/2f3c5964200f49bb83eaa4ea754ebd22 2024-11-19T11:14:46,446 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/6f793671b6874fc0834804c14d9d6487 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/C/6f793671b6874fc0834804c14d9d6487 2024-11-19T11:14:46,453 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/recovered.edits/531.seqid to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f/recovered.edits/531.seqid 2024-11-19T11:14:46,458 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/9cbe0c20892be91e1583baaaf6ed665f 2024-11-19T11:14:46,458 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T11:14:46,468 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:14:46,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-19T11:14:46,483 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T11:14:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T11:14:46,528 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T11:14:46,530 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:14:46,531 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T11:14:46,531 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732014886531"}]},"ts":"9223372036854775807"} 2024-11-19T11:14:46,537 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T11:14:46,537 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9cbe0c20892be91e1583baaaf6ed665f, NAME => 'TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T11:14:46,537 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T11:14:46,538 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732014886537"}]},"ts":"9223372036854775807"} 2024-11-19T11:14:46,541 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T11:14:46,543 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:14:46,545 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 150 msec 2024-11-19T11:14:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T11:14:46,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T11:14:46,720 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:14:46,734 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;16c88966c0af:35833-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1969214525_22 at /127.0.0.1:34152 [Waiting for operation #383] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_OPEN_REGION-regionserver/16c88966c0af:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-855594354_22 at /127.0.0.1:34256 [Waiting for operation #353] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=464 (was 272) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11845 (was 12466) 2024-11-19T11:14:46,744 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=464, ProcessCount=11, AvailableMemoryMB=11845 2024-11-19T11:14:46,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T11:14:46,747 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:14:46,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T11:14:46,750 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T11:14:46,750 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:46,750 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-19T11:14:46,751 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T11:14:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-19T11:14:46,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741956_1132 (size=1061) 2024-11-19T11:14:46,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-19T11:14:47,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-19T11:14:47,163 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094 2024-11-19T11:14:47,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741957_1133 (size=53) 2024-11-19T11:14:47,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:14:47,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing efdfe7abed14b673b10bbcdae68405c7, disabling compactions & flushes 2024-11-19T11:14:47,173 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:47,173 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:47,173 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. after waiting 0 ms 2024-11-19T11:14:47,173 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:47,173 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:47,173 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for efdfe7abed14b673b10bbcdae68405c7: Waiting for close lock at 1732014887172Disabling compacts and flushes for region at 1732014887172Disabling writes for close at 1732014887173 (+1 ms)Writing region close event to WAL at 1732014887173Closed at 1732014887173 2024-11-19T11:14:47,174 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T11:14:47,175 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732014887174"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732014887174"}]},"ts":"1732014887174"} 2024-11-19T11:14:47,176 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T11:14:47,178 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T11:14:47,178 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014887178"}]},"ts":"1732014887178"} 2024-11-19T11:14:47,180 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T11:14:47,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efdfe7abed14b673b10bbcdae68405c7, ASSIGN}] 2024-11-19T11:14:47,182 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efdfe7abed14b673b10bbcdae68405c7, ASSIGN 2024-11-19T11:14:47,183 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=efdfe7abed14b673b10bbcdae68405c7, ASSIGN; state=OFFLINE, location=16c88966c0af,35833,1732014855640; forceNewPlan=false, retain=false 2024-11-19T11:14:47,334 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=efdfe7abed14b673b10bbcdae68405c7, regionState=OPENING, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:14:47,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=efdfe7abed14b673b10bbcdae68405c7, ASSIGN because future has completed 2024-11-19T11:14:47,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640}] 2024-11-19T11:14:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-19T11:14:47,498 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:47,498 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} 2024-11-19T11:14:47,499 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:47,499 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:14:47,500 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:47,500 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:47,502 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:47,504 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:14:47,505 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efdfe7abed14b673b10bbcdae68405c7 columnFamilyName A 2024-11-19T11:14:47,505 DEBUG [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:47,506 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] regionserver.HStore(327): Store=efdfe7abed14b673b10bbcdae68405c7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:14:47,506 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:47,507 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:14:47,507 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efdfe7abed14b673b10bbcdae68405c7 columnFamilyName B 2024-11-19T11:14:47,507 DEBUG [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:47,508 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] regionserver.HStore(327): Store=efdfe7abed14b673b10bbcdae68405c7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:14:47,508 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:47,509 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:14:47,509 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efdfe7abed14b673b10bbcdae68405c7 columnFamilyName C 2024-11-19T11:14:47,509 DEBUG [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:47,509 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] regionserver.HStore(327): Store=efdfe7abed14b673b10bbcdae68405c7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:14:47,509 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:47,510 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:47,510 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:47,510 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:47,511 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:47,511 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:47,512 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T11:14:47,513 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:47,515 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T11:14:47,516 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened efdfe7abed14b673b10bbcdae68405c7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64860515, jitterRate=-0.033503010869026184}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T11:14:47,516 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:47,517 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for efdfe7abed14b673b10bbcdae68405c7: Running coprocessor pre-open hook at 1732014887500Writing region info on filesystem at 1732014887500Initializing all the Stores at 1732014887501 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014887501Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014887502 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014887502Cleaning up temporary data from old regions at 1732014887511 (+9 ms)Running coprocessor post-open hooks at 1732014887516 (+5 ms)Region opened successfully at 1732014887517 (+1 ms) 2024-11-19T11:14:47,518 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., pid=32, masterSystemTime=1732014887489 2024-11-19T11:14:47,520 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:47,520 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:47,520 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=efdfe7abed14b673b10bbcdae68405c7, regionState=OPEN, openSeqNum=2, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:14:47,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 because future has completed 2024-11-19T11:14:47,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-19T11:14:47,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 in 187 msec 2024-11-19T11:14:47,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-19T11:14:47,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efdfe7abed14b673b10bbcdae68405c7, ASSIGN in 345 msec 2024-11-19T11:14:47,529 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T11:14:47,529 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014887529"}]},"ts":"1732014887529"} 2024-11-19T11:14:47,531 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T11:14:47,532 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T11:14:47,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 785 msec 2024-11-19T11:14:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-19T11:14:47,889 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:14:47,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@244f1d4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:47,892 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:14:47,893 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:14:47,896 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:14:47,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:14:47,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:14:47,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5af5b370, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:47,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:14:47,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:14:47,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:47,899 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:14:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cedc1ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:47,900 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:14:47,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:14:47,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:14:47,903 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:14:47,904 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T11:14:47,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 16c88966c0af,41441,1732014854424 2024-11-19T11:14:47,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@17eb0dc9 2024-11-19T11:14:47,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T11:14:47,906 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T11:14:47,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T11:14:47,910 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:14:47,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T11:14:47,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741958_1134 (size=1097) 2024-11-19T11:14:48,332 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T11:14:48,332 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T11:14:48,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T11:14:48,349 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efdfe7abed14b673b10bbcdae68405c7, REOPEN/MOVE}] 2024-11-19T11:14:48,349 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efdfe7abed14b673b10bbcdae68405c7, REOPEN/MOVE 2024-11-19T11:14:48,350 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=efdfe7abed14b673b10bbcdae68405c7, regionState=CLOSING, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:14:48,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=efdfe7abed14b673b10bbcdae68405c7, REOPEN/MOVE because future has completed 2024-11-19T11:14:48,352 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T11:14:48,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640}] 2024-11-19T11:14:48,506 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:48,506 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T11:14:48,506 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing efdfe7abed14b673b10bbcdae68405c7, disabling compactions & flushes 2024-11-19T11:14:48,506 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:48,506 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:48,507 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. after waiting 1 ms 2024-11-19T11:14:48,507 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:48,514 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T11:14:48,515 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:48,515 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for efdfe7abed14b673b10bbcdae68405c7: Waiting for close lock at 1732014888506Running coprocessor pre-close hooks at 1732014888506Disabling compacts and flushes for region at 1732014888506Disabling writes for close at 1732014888507 (+1 ms)Writing region close event to WAL at 1732014888507Running coprocessor post-close hooks at 1732014888515 (+8 ms)Closed at 1732014888515 2024-11-19T11:14:48,515 WARN [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: efdfe7abed14b673b10bbcdae68405c7 to self. 2024-11-19T11:14:48,518 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:48,518 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=efdfe7abed14b673b10bbcdae68405c7, regionState=CLOSED 2024-11-19T11:14:48,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 because future has completed 2024-11-19T11:14:48,524 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-19T11:14:48,524 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 in 170 msec 2024-11-19T11:14:48,526 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=efdfe7abed14b673b10bbcdae68405c7, REOPEN/MOVE; state=CLOSED, location=16c88966c0af,35833,1732014855640; forceNewPlan=false, retain=true 2024-11-19T11:14:48,676 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=efdfe7abed14b673b10bbcdae68405c7, regionState=OPENING, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:14:48,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=efdfe7abed14b673b10bbcdae68405c7, REOPEN/MOVE because future has completed 2024-11-19T11:14:48,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640}] 2024-11-19T11:14:48,837 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:48,838 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} 2024-11-19T11:14:48,838 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:48,838 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:14:48,838 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:48,838 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:48,844 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:48,845 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:14:48,852 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efdfe7abed14b673b10bbcdae68405c7 columnFamilyName A 2024-11-19T11:14:48,854 DEBUG [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:48,855 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] regionserver.HStore(327): Store=efdfe7abed14b673b10bbcdae68405c7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:14:48,855 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:48,856 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:14:48,856 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efdfe7abed14b673b10bbcdae68405c7 columnFamilyName B 2024-11-19T11:14:48,856 DEBUG [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:48,856 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] regionserver.HStore(327): Store=efdfe7abed14b673b10bbcdae68405c7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:14:48,856 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:48,857 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:14:48,857 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efdfe7abed14b673b10bbcdae68405c7 columnFamilyName C 2024-11-19T11:14:48,857 DEBUG [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:48,858 INFO [StoreOpener-efdfe7abed14b673b10bbcdae68405c7-1 {}] regionserver.HStore(327): Store=efdfe7abed14b673b10bbcdae68405c7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:14:48,858 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:48,858 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:48,858 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:48,859 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:48,860 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:48,860 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:48,861 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T11:14:48,862 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:48,863 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened efdfe7abed14b673b10bbcdae68405c7; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65434818, jitterRate=-0.024945229291915894}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T11:14:48,863 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:48,864 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for efdfe7abed14b673b10bbcdae68405c7: Running coprocessor pre-open hook at 1732014888839Writing region info on filesystem at 1732014888839Initializing all the Stores at 1732014888840 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014888840Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014888840Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014888840Cleaning up temporary data from old regions at 1732014888860 (+20 ms)Running coprocessor post-open hooks at 1732014888863 (+3 ms)Region opened successfully at 1732014888864 (+1 ms) 2024-11-19T11:14:48,864 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., pid=37, masterSystemTime=1732014888832 2024-11-19T11:14:48,866 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:48,866 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:48,867 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=efdfe7abed14b673b10bbcdae68405c7, regionState=OPEN, openSeqNum=5, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:14:48,868 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 because future has completed 2024-11-19T11:14:48,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-19T11:14:48,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 in 190 msec 2024-11-19T11:14:48,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-19T11:14:48,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efdfe7abed14b673b10bbcdae68405c7, REOPEN/MOVE in 522 msec 2024-11-19T11:14:48,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-19T11:14:48,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 533 msec 2024-11-19T11:14:48,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 965 msec 2024-11-19T11:14:48,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-19T11:14:48,882 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:14:48,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b4ffc72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,883 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:14:48,883 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:14:48,884 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:14:48,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:14:48,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:14:48,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@101c089c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:14:48,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:14:48,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:48,886 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:14:48,887 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58d1d783, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,887 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,887 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:14:48,887 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:14:48,888 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:14:48,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:14:48,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:14:48,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33d92e8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:14:48,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:14:48,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:48,890 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:14:48,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a45b179, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,891 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,891 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:14:48,892 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:14:48,892 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:14:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:14:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:14:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555ce284, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:14:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:14:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:48,894 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:14:48,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f5751f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@208b436a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,896 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:14:48,896 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:14:48,896 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:14:48,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:14:48,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:14:48,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@704b7caf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:14:48,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:14:48,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:48,898 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:14:48,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4440b729, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349085e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,899 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:14:48,899 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:14:48,900 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:14:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:14:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:14:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ebeb9b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:14:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:14:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:48,902 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:14:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e2ada91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26dcfede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,904 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:14:48,904 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:14:48,905 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:14:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:14:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:14:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe0481d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:14:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:14:48,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:48,907 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:14:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@313af2bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d21c225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,908 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:14:48,908 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:14:48,909 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:14:48,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:14:48,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:14:48,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761a8232, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:14:48,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:14:48,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:48,910 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:14:48,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7203a827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a92e196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:14:48,912 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:14:48,913 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:14:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:14:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:14:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bfb8bf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:14:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:14:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:48,915 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:14:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7825eb66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3806f988, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,916 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:14:48,916 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:14:48,917 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:14:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:14:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:14:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef84692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:14:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:14:48,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:14:48,918 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:14:48,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cfa90e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:14:48,920 DEBUG [Thread-670 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:14:48,920 DEBUG [Thread-679 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:14:48,920 DEBUG [Thread-681 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:14:48,921 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:14:48,921 DEBUG [Thread-677 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:14:48,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:14:48,921 DEBUG [Thread-666 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:14:48,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:14:48,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:14:48,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:14:48,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:14:48,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:14:48,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:14:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-19T11:14:48,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:14:48,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:14:48,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:14:48,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:14:48,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:14:48,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:14:48,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:14:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T11:14:48,925 DEBUG [Thread-672 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:14:48,925 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:14:48,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:14:48,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:14:48,926 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:14:48,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:14:48,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:14:48,927 DEBUG [Thread-664 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:14:48,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:14:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:14:48,929 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:14:48,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:14:48,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:14:48,930 DEBUG [Thread-668 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:14:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:14:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:14:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:14:48,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:14:48,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:14:48,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:14:48,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:14:48,932 DEBUG [Thread-675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:14:48,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:14:48,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:14:48,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:14:48,934 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:14:48,934 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:14:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:14:48,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:14:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:48,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efdfe7abed14b673b10bbcdae68405c7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T11:14:48,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=A 2024-11-19T11:14:48,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:48,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=B 2024-11-19T11:14:48,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:48,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=C 2024-11-19T11:14:48,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:48,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195000da6bb11d43ce9546cef4a2439f9b_efdfe7abed14b673b10bbcdae68405c7 is 50, key is test_row_0/A:col10/1732014888941/Put/seqid=0 2024-11-19T11:14:49,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741959_1135 (size=14594) 2024-11-19T11:14:49,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014949005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:49,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014949007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:49,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014949007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:49,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014949009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:49,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014949019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T11:14:49,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-19T11:14:49,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:49,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:49,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:49,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:49,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:49,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:49,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014949128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:49,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:49,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014949128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:49,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:49,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:49,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:49,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014949129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:49,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:49,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:49,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014949129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:49,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:49,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:49,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014949131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:49,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-19T11:14:49,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:49,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:49,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:49,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:49,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:49,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:49,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T11:14:49,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:49,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:49,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014949338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:49,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014949338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:49,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:49,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014949339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:49,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:49,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014949339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:49,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:49,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:49,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014949339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:49,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-19T11:14:49,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:49,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:49,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:49,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:49,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:49,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:49,415 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:49,421 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195000da6bb11d43ce9546cef4a2439f9b_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195000da6bb11d43ce9546cef4a2439f9b_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:49,423 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/20e00b82b0744d92a1048fade156fd44, store: [table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:14:49,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/20e00b82b0744d92a1048fade156fd44 is 175, key is test_row_0/A:col10/1732014888941/Put/seqid=0 2024-11-19T11:14:49,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741960_1136 (size=39549) 2024-11-19T11:14:49,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-19T11:14:49,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:49,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:49,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:49,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:49,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T11:14:49,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014949648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:49,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:49,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014949648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:49,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:49,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:49,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014949648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:49,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:49,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:49,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014949648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:49,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014949648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:49,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-19T11:14:49,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:49,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:49,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:49,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:49,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:49,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-19T11:14:49,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:49,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:49,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:49,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:49,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:49,858 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/20e00b82b0744d92a1048fade156fd44 2024-11-19T11:14:49,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/42e0df460dd942529ec0fb1e7a5cbcc8 is 50, key is test_row_0/B:col10/1732014888941/Put/seqid=0 2024-11-19T11:14:49,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741961_1137 (size=12001) 2024-11-19T11:14:49,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/42e0df460dd942529ec0fb1e7a5cbcc8 2024-11-19T11:14:49,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/a550cf7e1066440ca5b93cc7906ee82e is 50, key is test_row_0/C:col10/1732014888941/Put/seqid=0 2024-11-19T11:14:49,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741962_1138 (size=12001) 2024-11-19T11:14:50,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-19T11:14:50,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:50,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:50,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:50,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:50,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:50,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T11:14:50,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-19T11:14:50,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:50,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:50,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:50,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:50,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:50,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:50,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:50,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014950158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:50,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:50,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:50,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014950158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:50,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014950159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:50,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:50,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014950159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:50,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:50,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014950159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:50,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:50,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:50,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:50,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-19T11:14:50,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:50,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:50,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:50,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:50,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:50,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:50,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/a550cf7e1066440ca5b93cc7906ee82e 2024-11-19T11:14:50,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/20e00b82b0744d92a1048fade156fd44 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/20e00b82b0744d92a1048fade156fd44 2024-11-19T11:14:50,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/20e00b82b0744d92a1048fade156fd44, entries=200, sequenceid=16, filesize=38.6 K 2024-11-19T11:14:50,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/42e0df460dd942529ec0fb1e7a5cbcc8 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/42e0df460dd942529ec0fb1e7a5cbcc8 2024-11-19T11:14:50,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/42e0df460dd942529ec0fb1e7a5cbcc8, entries=150, sequenceid=16, filesize=11.7 K 2024-11-19T11:14:50,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/a550cf7e1066440ca5b93cc7906ee82e as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/a550cf7e1066440ca5b93cc7906ee82e 2024-11-19T11:14:50,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/a550cf7e1066440ca5b93cc7906ee82e, entries=150, sequenceid=16, filesize=11.7 K 2024-11-19T11:14:50,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for efdfe7abed14b673b10bbcdae68405c7 in 1441ms, sequenceid=16, compaction requested=false 2024-11-19T11:14:50,395 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T11:14:50,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:14:50,453 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T11:14:50,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-19T11:14:50,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:50,468 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing efdfe7abed14b673b10bbcdae68405c7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T11:14:50,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=A 2024-11-19T11:14:50,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:50,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=B 2024-11-19T11:14:50,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:50,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=C 2024-11-19T11:14:50,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:50,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198b12024780bc4ab9937c0a73d012cdec_efdfe7abed14b673b10bbcdae68405c7 is 50, key is test_row_0/A:col10/1732014889004/Put/seqid=0 2024-11-19T11:14:50,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741963_1139 (size=12154) 2024-11-19T11:14:50,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,516 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198b12024780bc4ab9937c0a73d012cdec_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198b12024780bc4ab9937c0a73d012cdec_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:50,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/7b35c853e3b04c8693c864d1525fd9d0, store: [table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:14:50,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/7b35c853e3b04c8693c864d1525fd9d0 is 175, key is test_row_0/A:col10/1732014889004/Put/seqid=0 2024-11-19T11:14:50,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741964_1140 (size=30955) 2024-11-19T11:14:50,541 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/7b35c853e3b04c8693c864d1525fd9d0 2024-11-19T11:14:50,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/6625c3da5905429bb72b53304ddda0be is 50, key is test_row_0/B:col10/1732014889004/Put/seqid=0 2024-11-19T11:14:50,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741965_1141 (size=12001) 2024-11-19T11:14:50,574 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/6625c3da5905429bb72b53304ddda0be 2024-11-19T11:14:50,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/2f02ceca12b646b3be1242227b6e28cf is 50, key is test_row_0/C:col10/1732014889004/Put/seqid=0 2024-11-19T11:14:50,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741966_1142 (size=12001) 2024-11-19T11:14:50,590 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/2f02ceca12b646b3be1242227b6e28cf 2024-11-19T11:14:50,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/7b35c853e3b04c8693c864d1525fd9d0 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/7b35c853e3b04c8693c864d1525fd9d0 2024-11-19T11:14:50,605 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/7b35c853e3b04c8693c864d1525fd9d0, entries=150, sequenceid=36, filesize=30.2 K 2024-11-19T11:14:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/6625c3da5905429bb72b53304ddda0be as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/6625c3da5905429bb72b53304ddda0be 2024-11-19T11:14:50,620 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/6625c3da5905429bb72b53304ddda0be, entries=150, sequenceid=36, filesize=11.7 K 2024-11-19T11:14:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/2f02ceca12b646b3be1242227b6e28cf as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/2f02ceca12b646b3be1242227b6e28cf 2024-11-19T11:14:50,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,627 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/2f02ceca12b646b3be1242227b6e28cf, entries=150, sequenceid=36, filesize=11.7 K 2024-11-19T11:14:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,629 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for efdfe7abed14b673b10bbcdae68405c7 in 160ms, sequenceid=36, compaction requested=false 2024-11-19T11:14:50,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:14:50,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:50,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-19T11:14:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-19T11:14:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-19T11:14:50,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7050 sec 2024-11-19T11:14:50,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.7140 sec 2024-11-19T11:14:50,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:50,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T11:14:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,068 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:14:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,069 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:14:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-19T11:14:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T11:14:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,073 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:14:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,074 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:14:51,074 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:14:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T11:14:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:51,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efdfe7abed14b673b10bbcdae68405c7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T11:14:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=A 2024-11-19T11:14:51,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:51,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=B 2024-11-19T11:14:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:51,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=C 2024-11-19T11:14:51,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:51,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T11:14:51,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:51,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:51,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119605333a8d3b24ab7961c1b531220e625_efdfe7abed14b673b10bbcdae68405c7 is 50, key is test_row_0/A:col10/1732014891226/Put/seqid=0 2024-11-19T11:14:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741968_1144 (size=24358) 2024-11-19T11:14:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:51,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1732014891228, max timestamp=1732014891288Num uniques 859; 2024-11-19T11:14:51,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1732014891228, max timestamp=1732014891288Num uniques 859; 2024-11-19T11:14:51,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1732014891228, max timestamp=1732014891288Num uniques 859; 2024-11-19T11:14:51,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-19T11:14:51,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-19T11:14:51,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-19T11:14:51,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014951278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:51,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014951296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:51,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014951298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:51,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014951299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014951300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:51,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T11:14:51,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:51,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:51,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:51,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:51,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T11:14:51,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014951397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:51,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:51,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014951407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:51,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:51,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014951408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:51,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:51,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:51,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014951408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:51,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014951408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:51,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:51,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:51,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T11:14:51,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:51,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:51,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:51,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:51,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:51,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014951607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:51,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:51,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014951617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:51,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014951618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014951618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014951618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:51,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:51,659 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:51,662 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119605333a8d3b24ab7961c1b531220e625_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119605333a8d3b24ab7961c1b531220e625_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:51,663 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/cf36185268204a2da92fbf5452fd4679, store: [table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:14:51,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/cf36185268204a2da92fbf5452fd4679 is 175, key is test_row_0/A:col10/1732014891226/Put/seqid=0 2024-11-19T11:14:51,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741967_1143 (size=73995) 2024-11-19T11:14:51,667 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=47, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/cf36185268204a2da92fbf5452fd4679 2024-11-19T11:14:51,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/4baf5a218787414f94399456f2521efd is 50, key is test_row_0/B:col10/1732014891226/Put/seqid=0 2024-11-19T11:14:51,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741969_1145 (size=12001) 2024-11-19T11:14:51,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/4baf5a218787414f94399456f2521efd 2024-11-19T11:14:51,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T11:14:51,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:51,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:51,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:51,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:51,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T11:14:51,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/36e01e666a0c4a2d81cb905b9f52c8a6 is 50, key is test_row_0/C:col10/1732014891226/Put/seqid=0 2024-11-19T11:14:51,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741970_1146 (size=12001) 2024-11-19T11:14:51,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/36e01e666a0c4a2d81cb905b9f52c8a6 2024-11-19T11:14:51,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/cf36185268204a2da92fbf5452fd4679 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/cf36185268204a2da92fbf5452fd4679 2024-11-19T11:14:51,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/cf36185268204a2da92fbf5452fd4679, entries=400, sequenceid=47, filesize=72.3 K 2024-11-19T11:14:51,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/4baf5a218787414f94399456f2521efd as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/4baf5a218787414f94399456f2521efd 2024-11-19T11:14:51,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/4baf5a218787414f94399456f2521efd, entries=150, sequenceid=47, filesize=11.7 K 2024-11-19T11:14:51,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/36e01e666a0c4a2d81cb905b9f52c8a6 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/36e01e666a0c4a2d81cb905b9f52c8a6 2024-11-19T11:14:51,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/36e01e666a0c4a2d81cb905b9f52c8a6, entries=150, sequenceid=47, filesize=11.7 K 2024-11-19T11:14:51,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=154.31 KB/158010 for efdfe7abed14b673b10bbcdae68405c7 in 528ms, sequenceid=47, compaction requested=true 2024-11-19T11:14:51,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:14:51,755 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:51,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efdfe7abed14b673b10bbcdae68405c7:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:14:51,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:51,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efdfe7abed14b673b10bbcdae68405c7:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:14:51,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:51,755 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:51,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efdfe7abed14b673b10bbcdae68405c7:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:14:51,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:51,757 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 144499 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:51,757 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): efdfe7abed14b673b10bbcdae68405c7/A is initiating minor compaction (all files) 2024-11-19T11:14:51,757 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efdfe7abed14b673b10bbcdae68405c7/A in TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:51,757 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/20e00b82b0744d92a1048fade156fd44, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/7b35c853e3b04c8693c864d1525fd9d0, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/cf36185268204a2da92fbf5452fd4679] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp, totalSize=141.1 K 2024-11-19T11:14:51,757 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:51,757 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. files: [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/20e00b82b0744d92a1048fade156fd44, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/7b35c853e3b04c8693c864d1525fd9d0, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/cf36185268204a2da92fbf5452fd4679] 2024-11-19T11:14:51,758 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20e00b82b0744d92a1048fade156fd44, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732014888941 2024-11-19T11:14:51,759 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b35c853e3b04c8693c864d1525fd9d0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732014889004 2024-11-19T11:14:51,761 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:51,761 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): efdfe7abed14b673b10bbcdae68405c7/B is initiating minor compaction (all files) 2024-11-19T11:14:51,761 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efdfe7abed14b673b10bbcdae68405c7/B in TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:51,762 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/42e0df460dd942529ec0fb1e7a5cbcc8, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/6625c3da5905429bb72b53304ddda0be, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/4baf5a218787414f94399456f2521efd] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp, totalSize=35.2 K 2024-11-19T11:14:51,762 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf36185268204a2da92fbf5452fd4679, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732014891184 2024-11-19T11:14:51,766 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 42e0df460dd942529ec0fb1e7a5cbcc8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732014888941 2024-11-19T11:14:51,766 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 6625c3da5905429bb72b53304ddda0be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732014889004 2024-11-19T11:14:51,767 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 4baf5a218787414f94399456f2521efd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732014891204 2024-11-19T11:14:51,781 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:14:51,783 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efdfe7abed14b673b10bbcdae68405c7#B#compaction#128 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:51,783 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/72a389ec6d27424381cd18f24294bac7 is 50, key is test_row_0/B:col10/1732014891226/Put/seqid=0 2024-11-19T11:14:51,801 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119d080d3db5ade488e819c669523a25733_efdfe7abed14b673b10bbcdae68405c7 store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:14:51,805 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119d080d3db5ade488e819c669523a25733_efdfe7abed14b673b10bbcdae68405c7, store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:14:51,806 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d080d3db5ade488e819c669523a25733_efdfe7abed14b673b10bbcdae68405c7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:14:51,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741971_1147 (size=12104) 2024-11-19T11:14:51,829 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/72a389ec6d27424381cd18f24294bac7 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/72a389ec6d27424381cd18f24294bac7 2024-11-19T11:14:51,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741972_1148 (size=4469) 2024-11-19T11:14:51,837 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efdfe7abed14b673b10bbcdae68405c7/B of efdfe7abed14b673b10bbcdae68405c7 into 72a389ec6d27424381cd18f24294bac7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:51,837 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:14:51,837 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., storeName=efdfe7abed14b673b10bbcdae68405c7/B, priority=13, startTime=1732014891755; duration=0sec 2024-11-19T11:14:51,837 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:51,837 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efdfe7abed14b673b10bbcdae68405c7:B 2024-11-19T11:14:51,837 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:14:51,838 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:14:51,839 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): efdfe7abed14b673b10bbcdae68405c7/C is initiating minor compaction (all files) 2024-11-19T11:14:51,839 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efdfe7abed14b673b10bbcdae68405c7/C in TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:51,839 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/a550cf7e1066440ca5b93cc7906ee82e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/2f02ceca12b646b3be1242227b6e28cf, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/36e01e666a0c4a2d81cb905b9f52c8a6] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp, totalSize=35.2 K 2024-11-19T11:14:51,839 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting a550cf7e1066440ca5b93cc7906ee82e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732014888941 2024-11-19T11:14:51,840 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f02ceca12b646b3be1242227b6e28cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732014889004 2024-11-19T11:14:51,840 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 36e01e666a0c4a2d81cb905b9f52c8a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732014891204 2024-11-19T11:14:51,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T11:14:51,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:51,844 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing efdfe7abed14b673b10bbcdae68405c7 3/3 column families, dataSize=154.31 KB heapSize=379.13 KB 2024-11-19T11:14:51,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=A 2024-11-19T11:14:51,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:51,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=B 2024-11-19T11:14:51,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:51,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=C 2024-11-19T11:14:51,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:51,855 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efdfe7abed14b673b10bbcdae68405c7#C#compaction#130 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:51,856 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/72b01cabc4cd4c919c4cc99c7b4f324d is 50, key is test_row_0/C:col10/1732014891226/Put/seqid=0 2024-11-19T11:14:51,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195143bdcb49d3464fbf3562f994439ff1_efdfe7abed14b673b10bbcdae68405c7 is 50, key is test_row_0/A:col10/1732014891292/Put/seqid=0 2024-11-19T11:14:51,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741973_1149 (size=12104) 2024-11-19T11:14:51,879 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/72b01cabc4cd4c919c4cc99c7b4f324d as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/72b01cabc4cd4c919c4cc99c7b4f324d 2024-11-19T11:14:51,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741974_1150 (size=12154) 2024-11-19T11:14:51,891 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efdfe7abed14b673b10bbcdae68405c7/C of efdfe7abed14b673b10bbcdae68405c7 into 72b01cabc4cd4c919c4cc99c7b4f324d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:51,891 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:14:51,891 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., storeName=efdfe7abed14b673b10bbcdae68405c7/C, priority=13, startTime=1732014891755; duration=0sec 2024-11-19T11:14:51,891 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:51,892 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efdfe7abed14b673b10bbcdae68405c7:C 2024-11-19T11:14:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:51,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:51,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014951937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:51,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:51,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014951937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:51,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:51,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014951939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:51,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:51,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014951940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:51,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:51,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014951940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:51,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:51,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:51,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:52,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:52,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014952047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:52,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014952047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:52,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:52,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014952058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:52,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014952058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014952058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T11:14:52,237 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efdfe7abed14b673b10bbcdae68405c7#A#compaction#129 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:52,238 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/edd5b08884134d74ab41fa38124a2e82 is 175, key is test_row_0/A:col10/1732014891226/Put/seqid=0 2024-11-19T11:14:52,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741975_1151 (size=31058) 2024-11-19T11:14:52,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014952257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:52,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:52,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014952257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:52,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:52,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014952268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:52,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:52,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014952268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:52,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014952268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:52,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:52,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:52,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:52,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:52,290 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195143bdcb49d3464fbf3562f994439ff1_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195143bdcb49d3464fbf3562f994439ff1_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:52,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/4f383e08229d451fa9279e32b6a14399, store: [table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:14:52,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/4f383e08229d451fa9279e32b6a14399 is 175, key is test_row_0/A:col10/1732014891292/Put/seqid=0 2024-11-19T11:14:52,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741976_1152 (size=30955) 2024-11-19T11:14:52,304 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=74, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/4f383e08229d451fa9279e32b6a14399 2024-11-19T11:14:52,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/7cb6f01298304a3cafd71b00480225b3 is 50, key is test_row_0/B:col10/1732014891292/Put/seqid=0 2024-11-19T11:14:52,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741977_1153 (size=12001) 2024-11-19T11:14:52,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014952567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:52,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014952577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:52,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:52,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014952578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014952578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014952578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:52,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:52,649 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/edd5b08884134d74ab41fa38124a2e82 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/edd5b08884134d74ab41fa38124a2e82 2024-11-19T11:14:52,656 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efdfe7abed14b673b10bbcdae68405c7/A of efdfe7abed14b673b10bbcdae68405c7 into edd5b08884134d74ab41fa38124a2e82(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:52,656 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:14:52,656 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., storeName=efdfe7abed14b673b10bbcdae68405c7/A, priority=13, startTime=1732014891755; duration=0sec 2024-11-19T11:14:52,656 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:52,656 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efdfe7abed14b673b10bbcdae68405c7:A 2024-11-19T11:14:52,729 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/7cb6f01298304a3cafd71b00480225b3 2024-11-19T11:14:52,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/13ec8655a70d435fa614053e45163d83 is 50, key is test_row_0/C:col10/1732014891292/Put/seqid=0 2024-11-19T11:14:52,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741978_1154 (size=12001) 2024-11-19T11:14:53,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:53,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014953077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:53,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:53,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:53,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:53,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:53,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014953088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:53,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:53,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014953088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:53,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:53,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014953088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:53,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:53,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:53,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014953088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:53,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:53,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:53,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:53,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:53,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:53,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:53,141 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/13ec8655a70d435fa614053e45163d83 2024-11-19T11:14:53,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/4f383e08229d451fa9279e32b6a14399 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/4f383e08229d451fa9279e32b6a14399 2024-11-19T11:14:53,150 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/4f383e08229d451fa9279e32b6a14399, entries=150, sequenceid=74, filesize=30.2 K 2024-11-19T11:14:53,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/7cb6f01298304a3cafd71b00480225b3 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/7cb6f01298304a3cafd71b00480225b3 2024-11-19T11:14:53,156 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/7cb6f01298304a3cafd71b00480225b3, entries=150, sequenceid=74, filesize=11.7 K 2024-11-19T11:14:53,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/13ec8655a70d435fa614053e45163d83 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/13ec8655a70d435fa614053e45163d83 2024-11-19T11:14:53,161 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/13ec8655a70d435fa614053e45163d83, entries=150, sequenceid=74, filesize=11.7 K 2024-11-19T11:14:53,162 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~379.08 KB/388176, currentSize=53.67 KB/54960 for efdfe7abed14b673b10bbcdae68405c7 in 1318ms, sequenceid=74, compaction requested=false 2024-11-19T11:14:53,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:14:53,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:53,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-19T11:14:53,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-19T11:14:53,166 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-19T11:14:53,166 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0900 sec 2024-11-19T11:14:53,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.0980 sec 2024-11-19T11:14:53,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T11:14:53,218 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:14:53,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:14:53,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-19T11:14:53,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T11:14:53,220 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:14:53,221 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:14:53,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:14:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T11:14:53,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T11:14:53,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:53,374 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing efdfe7abed14b673b10bbcdae68405c7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T11:14:53,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=A 2024-11-19T11:14:53,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:53,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=B 2024-11-19T11:14:53,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:53,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=C 2024-11-19T11:14:53,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:53,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193053e3e8ad0d4c21b9723ab15206c154_efdfe7abed14b673b10bbcdae68405c7 is 50, key is test_row_0/A:col10/1732014891936/Put/seqid=0 2024-11-19T11:14:53,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741979_1155 (size=12154) 2024-11-19T11:14:53,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T11:14:53,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:53,796 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193053e3e8ad0d4c21b9723ab15206c154_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193053e3e8ad0d4c21b9723ab15206c154_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:53,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/3cc84bd51f9e43bcab23936b75456c79, store: [table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:14:53,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/3cc84bd51f9e43bcab23936b75456c79 is 175, key is test_row_0/A:col10/1732014891936/Put/seqid=0 2024-11-19T11:14:53,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741980_1156 (size=30955) 2024-11-19T11:14:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T11:14:54,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:54,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:54,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014954118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:54,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014954119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:54,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732014894093, max timestamp=1732014894121Num uniques 859; 2024-11-19T11:14:54,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732014894093, max timestamp=1732014894121Num uniques 859; 2024-11-19T11:14:54,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:54,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T11:14:54,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732014894093, max timestamp=1732014894121Num uniques 859; 2024-11-19T11:14:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014954120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:54,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T11:14:54,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T11:14:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:54,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014954155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:54,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:54,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014954156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:54,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:54,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:54,201 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=87, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/3cc84bd51f9e43bcab23936b75456c79 2024-11-19T11:14:54,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/a465b825bb86409eb40907212ad45d4d is 50, key is test_row_0/B:col10/1732014891936/Put/seqid=0 2024-11-19T11:14:54,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741981_1157 (size=12001) 2024-11-19T11:14:54,217 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/a465b825bb86409eb40907212ad45d4d 2024-11-19T11:14:54,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/03514653ef44444ba64e8c218641b791 is 50, key is test_row_0/C:col10/1732014891936/Put/seqid=0 2024-11-19T11:14:54,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014954228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:54,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:54,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014954228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:54,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:54,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014954228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:54,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:54,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741982_1158 (size=12001) 2024-11-19T11:14:54,234 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/03514653ef44444ba64e8c218641b791 2024-11-19T11:14:54,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/3cc84bd51f9e43bcab23936b75456c79 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/3cc84bd51f9e43bcab23936b75456c79 2024-11-19T11:14:54,248 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/3cc84bd51f9e43bcab23936b75456c79, entries=150, sequenceid=87, filesize=30.2 K 2024-11-19T11:14:54,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/a465b825bb86409eb40907212ad45d4d as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/a465b825bb86409eb40907212ad45d4d 2024-11-19T11:14:54,258 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/a465b825bb86409eb40907212ad45d4d, entries=150, sequenceid=87, filesize=11.7 K 2024-11-19T11:14:54,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/03514653ef44444ba64e8c218641b791 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/03514653ef44444ba64e8c218641b791 2024-11-19T11:14:54,265 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/03514653ef44444ba64e8c218641b791, entries=150, sequenceid=87, filesize=11.7 K 2024-11-19T11:14:54,266 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for efdfe7abed14b673b10bbcdae68405c7 in 892ms, sequenceid=87, compaction requested=true 2024-11-19T11:14:54,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:14:54,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:54,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-19T11:14:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-19T11:14:54,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-19T11:14:54,273 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0470 sec 2024-11-19T11:14:54,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.0560 sec 2024-11-19T11:14:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:54,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efdfe7abed14b673b10bbcdae68405c7 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-19T11:14:54,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=A 2024-11-19T11:14:54,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:54,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=B 2024-11-19T11:14:54,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:54,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=C 2024-11-19T11:14:54,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:54,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c04800c0dc1c4552b29bf18f878e1a3e_efdfe7abed14b673b10bbcdae68405c7 is 50, key is test_row_0/A:col10/1732014894127/Put/seqid=0 2024-11-19T11:14:54,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014954300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:54,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:54,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014954306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:54,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:54,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741983_1159 (size=14594) 2024-11-19T11:14:54,355 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T11:14:54,358 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:14:54,358 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:14:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-19T11:14:54,360 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c04800c0dc1c4552b29bf18f878e1a3e_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c04800c0dc1c4552b29bf18f878e1a3e_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T11:14:54,360 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:14:54,361 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:14:54,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:14:54,361 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/06e205d592cf4609aa4dbf1d9bdef969, store: [table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:14:54,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/06e205d592cf4609aa4dbf1d9bdef969 is 175, key is test_row_0/A:col10/1732014894127/Put/seqid=0 2024-11-19T11:14:54,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741984_1160 (size=39549) 2024-11-19T11:14:54,368 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/06e205d592cf4609aa4dbf1d9bdef969 2024-11-19T11:14:54,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/db64e9ab44604ba9b001e42530f1d2a7 is 50, key is test_row_0/B:col10/1732014894127/Put/seqid=0 2024-11-19T11:14:54,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741985_1161 (size=12001) 2024-11-19T11:14:54,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014954418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:54,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:54,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014954418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:54,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:54,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:54,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014954437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:54,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:54,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014954437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:54,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:54,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:54,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014954438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:54,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T11:14:54,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T11:14:54,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:54,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:54,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:54,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:54,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:54,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014954628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:54,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:54,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014954628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:54,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:54,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:54,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T11:14:54,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:54,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:54,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:54,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:54,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T11:14:54,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014954747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:54,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014954747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:54,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014954747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:54,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/db64e9ab44604ba9b001e42530f1d2a7 2024-11-19T11:14:54,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/51188b3feba94b468caf448d5ee54682 is 50, key is test_row_0/C:col10/1732014894127/Put/seqid=0 2024-11-19T11:14:54,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741986_1162 (size=12001) 2024-11-19T11:14:54,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T11:14:54,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:54,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:54,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:54,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:54,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:54,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014954938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:54,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:54,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014954947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:54,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T11:14:54,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:54,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:54,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:54,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:54,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T11:14:55,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T11:14:55,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:55,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:55,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:55,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:55,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:55,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/51188b3feba94b468caf448d5ee54682 2024-11-19T11:14:55,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/06e205d592cf4609aa4dbf1d9bdef969 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/06e205d592cf4609aa4dbf1d9bdef969 2024-11-19T11:14:55,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/06e205d592cf4609aa4dbf1d9bdef969, entries=200, sequenceid=122, filesize=38.6 K 2024-11-19T11:14:55,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/db64e9ab44604ba9b001e42530f1d2a7 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/db64e9ab44604ba9b001e42530f1d2a7 2024-11-19T11:14:55,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/db64e9ab44604ba9b001e42530f1d2a7, entries=150, sequenceid=122, filesize=11.7 K 2024-11-19T11:14:55,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/51188b3feba94b468caf448d5ee54682 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/51188b3feba94b468caf448d5ee54682 2024-11-19T11:14:55,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/51188b3feba94b468caf448d5ee54682, entries=150, sequenceid=122, filesize=11.7 K 2024-11-19T11:14:55,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for efdfe7abed14b673b10bbcdae68405c7 in 953ms, sequenceid=122, compaction requested=true 2024-11-19T11:14:55,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:14:55,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efdfe7abed14b673b10bbcdae68405c7:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:14:55,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:55,233 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:14:55,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efdfe7abed14b673b10bbcdae68405c7:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:14:55,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:55,233 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:14:55,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efdfe7abed14b673b10bbcdae68405c7:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:14:55,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:55,236 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:14:55,236 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): efdfe7abed14b673b10bbcdae68405c7/A is initiating minor compaction (all files) 2024-11-19T11:14:55,237 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efdfe7abed14b673b10bbcdae68405c7/A in TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:55,237 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/edd5b08884134d74ab41fa38124a2e82, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/4f383e08229d451fa9279e32b6a14399, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/3cc84bd51f9e43bcab23936b75456c79, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/06e205d592cf4609aa4dbf1d9bdef969] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp, totalSize=129.4 K 2024-11-19T11:14:55,237 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:55,237 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. files: [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/edd5b08884134d74ab41fa38124a2e82, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/4f383e08229d451fa9279e32b6a14399, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/3cc84bd51f9e43bcab23936b75456c79, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/06e205d592cf4609aa4dbf1d9bdef969] 2024-11-19T11:14:55,237 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:14:55,237 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): efdfe7abed14b673b10bbcdae68405c7/B is initiating minor compaction (all files) 2024-11-19T11:14:55,237 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efdfe7abed14b673b10bbcdae68405c7/B in TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:55,237 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/72a389ec6d27424381cd18f24294bac7, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/7cb6f01298304a3cafd71b00480225b3, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/a465b825bb86409eb40907212ad45d4d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/db64e9ab44604ba9b001e42530f1d2a7] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp, totalSize=47.0 K 2024-11-19T11:14:55,238 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 72a389ec6d27424381cd18f24294bac7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732014891204 2024-11-19T11:14:55,239 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting edd5b08884134d74ab41fa38124a2e82, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732014891204 2024-11-19T11:14:55,239 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cb6f01298304a3cafd71b00480225b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732014891292 2024-11-19T11:14:55,239 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f383e08229d451fa9279e32b6a14399, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732014891292 2024-11-19T11:14:55,240 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting a465b825bb86409eb40907212ad45d4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732014891936 2024-11-19T11:14:55,240 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cc84bd51f9e43bcab23936b75456c79, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732014891936 2024-11-19T11:14:55,240 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting db64e9ab44604ba9b001e42530f1d2a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732014894127 2024-11-19T11:14:55,241 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06e205d592cf4609aa4dbf1d9bdef969, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732014894127 2024-11-19T11:14:55,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:55,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efdfe7abed14b673b10bbcdae68405c7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T11:14:55,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=A 2024-11-19T11:14:55,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:55,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=B 2024-11-19T11:14:55,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:55,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=C 2024-11-19T11:14:55,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:55,268 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efdfe7abed14b673b10bbcdae68405c7#B#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:55,268 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:14:55,269 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/3551ffe615a94c35a88328300766ec92 is 50, key is test_row_0/B:col10/1732014894127/Put/seqid=0 2024-11-19T11:14:55,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T11:14:55,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:55,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:55,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:55,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:55,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:55,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:55,285 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119a7c7ce09bf4e4cbe98da8d0938a3f312_efdfe7abed14b673b10bbcdae68405c7 store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:14:55,288 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119a7c7ce09bf4e4cbe98da8d0938a3f312_efdfe7abed14b673b10bbcdae68405c7, store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:14:55,288 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a7c7ce09bf4e4cbe98da8d0938a3f312_efdfe7abed14b673b10bbcdae68405c7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:14:55,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119eed80fef64f24a628e49e7629f0bc0ac_efdfe7abed14b673b10bbcdae68405c7 is 50, key is test_row_0/A:col10/1732014895258/Put/seqid=0 2024-11-19T11:14:55,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741987_1163 (size=12241) 2024-11-19T11:14:55,319 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/3551ffe615a94c35a88328300766ec92 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/3551ffe615a94c35a88328300766ec92 2024-11-19T11:14:55,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014895305, max timestamp=1732014895317Num uniques 859; 2024-11-19T11:14:55,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:14:55,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014895305, max timestamp=1732014895317Num uniques 859; 2024-11-19T11:14:55,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:14:55,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732014895286, max timestamp=1732014895317Num uniques 859; 2024-11-19T11:14:55,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T11:14:55,330 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efdfe7abed14b673b10bbcdae68405c7/B of efdfe7abed14b673b10bbcdae68405c7 into 3551ffe615a94c35a88328300766ec92(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:55,330 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:14:55,330 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., storeName=efdfe7abed14b673b10bbcdae68405c7/B, priority=12, startTime=1732014895233; duration=0sec 2024-11-19T11:14:55,330 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:55,330 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efdfe7abed14b673b10bbcdae68405c7:B 2024-11-19T11:14:55,330 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:14:55,334 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:14:55,334 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): efdfe7abed14b673b10bbcdae68405c7/C is initiating minor compaction (all files) 2024-11-19T11:14:55,334 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efdfe7abed14b673b10bbcdae68405c7/C in TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:55,334 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/72b01cabc4cd4c919c4cc99c7b4f324d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/13ec8655a70d435fa614053e45163d83, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/03514653ef44444ba64e8c218641b791, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/51188b3feba94b468caf448d5ee54682] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp, totalSize=47.0 K 2024-11-19T11:14:55,335 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 72b01cabc4cd4c919c4cc99c7b4f324d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732014891204 2024-11-19T11:14:55,335 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 13ec8655a70d435fa614053e45163d83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732014891292 2024-11-19T11:14:55,335 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 03514653ef44444ba64e8c218641b791, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732014891936 2024-11-19T11:14:55,336 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 51188b3feba94b468caf448d5ee54682, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732014894127 2024-11-19T11:14:55,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741988_1164 (size=4469) 2024-11-19T11:14:55,347 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efdfe7abed14b673b10bbcdae68405c7#A#compaction#141 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:55,348 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/7521958460c44954b4d36e316352e710 is 175, key is test_row_0/A:col10/1732014894127/Put/seqid=0 2024-11-19T11:14:55,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741989_1165 (size=14744) 2024-11-19T11:14:55,358 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:55,364 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119eed80fef64f24a628e49e7629f0bc0ac_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119eed80fef64f24a628e49e7629f0bc0ac_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:55,366 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/19462fd78f384776bb9bb95e87126cb4, store: [table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:14:55,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/19462fd78f384776bb9bb95e87126cb4 is 175, key is test_row_0/A:col10/1732014895258/Put/seqid=0 2024-11-19T11:14:55,367 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efdfe7abed14b673b10bbcdae68405c7#C#compaction#143 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:55,368 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/1ebd602cbbba4cf8b363eddd35ddada1 is 50, key is test_row_0/C:col10/1732014894127/Put/seqid=0 2024-11-19T11:14:55,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741990_1166 (size=31195) 2024-11-19T11:14:55,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:55,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014955371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:55,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:55,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:55,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:55,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741991_1167 (size=39699) 2024-11-19T11:14:55,391 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/19462fd78f384776bb9bb95e87126cb4 2024-11-19T11:14:55,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:55,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014955381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:55,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014955381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:55,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:55,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:55,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:55,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:55,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:55,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:55,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741992_1168 (size=12241) 2024-11-19T11:14:55,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/3bcb6fde91f040d198146b6e462f1ead is 50, key is test_row_0/B:col10/1732014895258/Put/seqid=0 2024-11-19T11:14:55,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741993_1169 (size=12101) 2024-11-19T11:14:55,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/3bcb6fde91f040d198146b6e462f1ead 2024-11-19T11:14:55,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/4f47ee358bb341df835fa15669b90314 is 50, key is test_row_0/C:col10/1732014895258/Put/seqid=0 2024-11-19T11:14:55,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741994_1170 (size=12101) 2024-11-19T11:14:55,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T11:14:55,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:55,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:55,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:55,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:55,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:55,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014955447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:55,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014955467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:55,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014955488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:55,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:55,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:55,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T11:14:55,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014955497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:55,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:55,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:55,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:55,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014955497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:55,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:55,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:55,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:55,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T11:14:55,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:55,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:55,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:55,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:55,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:55,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014955697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:55,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:55,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:55,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:55,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014955707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:55,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:55,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:55,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:55,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014955707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:55,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:55,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:55,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:55,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T11:14:55,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:55,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:55,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:55,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:55,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:55,779 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/7521958460c44954b4d36e316352e710 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/7521958460c44954b4d36e316352e710 2024-11-19T11:14:55,784 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efdfe7abed14b673b10bbcdae68405c7/A of efdfe7abed14b673b10bbcdae68405c7 into 7521958460c44954b4d36e316352e710(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:55,784 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:14:55,784 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., storeName=efdfe7abed14b673b10bbcdae68405c7/A, priority=12, startTime=1732014895233; duration=0sec 2024-11-19T11:14:55,784 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:55,784 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efdfe7abed14b673b10bbcdae68405c7:A 2024-11-19T11:14:55,798 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/1ebd602cbbba4cf8b363eddd35ddada1 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/1ebd602cbbba4cf8b363eddd35ddada1 2024-11-19T11:14:55,802 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efdfe7abed14b673b10bbcdae68405c7/C of efdfe7abed14b673b10bbcdae68405c7 into 1ebd602cbbba4cf8b363eddd35ddada1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:55,802 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:14:55,802 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., storeName=efdfe7abed14b673b10bbcdae68405c7/C, priority=12, startTime=1732014895233; duration=0sec 2024-11-19T11:14:55,803 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:55,803 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efdfe7abed14b673b10bbcdae68405c7:C 2024-11-19T11:14:55,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/4f47ee358bb341df835fa15669b90314 2024-11-19T11:14:55,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/19462fd78f384776bb9bb95e87126cb4 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/19462fd78f384776bb9bb95e87126cb4 2024-11-19T11:14:55,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/19462fd78f384776bb9bb95e87126cb4, entries=200, sequenceid=133, filesize=38.8 K 2024-11-19T11:14:55,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/3bcb6fde91f040d198146b6e462f1ead as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/3bcb6fde91f040d198146b6e462f1ead 2024-11-19T11:14:55,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/3bcb6fde91f040d198146b6e462f1ead, entries=150, sequenceid=133, filesize=11.8 K 2024-11-19T11:14:55,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/4f47ee358bb341df835fa15669b90314 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/4f47ee358bb341df835fa15669b90314 2024-11-19T11:14:55,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/4f47ee358bb341df835fa15669b90314, entries=150, sequenceid=133, filesize=11.8 K 2024-11-19T11:14:55,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=136.42 KB/139690 for efdfe7abed14b673b10bbcdae68405c7 in 597ms, sequenceid=133, compaction requested=false 2024-11-19T11:14:55,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:14:55,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T11:14:55,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:55,897 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing efdfe7abed14b673b10bbcdae68405c7 3/3 column families, dataSize=136.42 KB heapSize=385.38 KB 2024-11-19T11:14:55,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=A 2024-11-19T11:14:55,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:55,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=B 2024-11-19T11:14:55,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:55,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=C 2024-11-19T11:14:55,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:55,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199147cfa1ca314702a5d1859db263e2b1_efdfe7abed14b673b10bbcdae68405c7 is 50, key is test_row_0/A:col10/1732014895378/Put/seqid=0 2024-11-19T11:14:55,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741995_1171 (size=12304) 2024-11-19T11:14:55,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:55,920 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199147cfa1ca314702a5d1859db263e2b1_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199147cfa1ca314702a5d1859db263e2b1_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:55,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/a2119629aae747128c043f4bc6c7cf46, store: [table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:14:55,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/a2119629aae747128c043f4bc6c7cf46 is 175, key is test_row_0/A:col10/1732014895378/Put/seqid=0 2024-11-19T11:14:55,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741996_1172 (size=31105) 2024-11-19T11:14:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:56,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:56,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014956023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:56,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014956028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:56,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014956028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:56,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014956137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014956137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:56,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:56,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014956137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:56,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:56,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:56,329 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/a2119629aae747128c043f4bc6c7cf46 2024-11-19T11:14:56,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/c7cb91d518e6487cb1f1896ae10e868b is 50, key is test_row_0/B:col10/1732014895378/Put/seqid=0 2024-11-19T11:14:56,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741997_1173 (size=12151) 2024-11-19T11:14:56,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014956347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:56,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014956347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:56,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014956347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:56,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:56,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014956457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:56,461 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_1' in efdfe7abed14b673b10bbcdae68405c7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:14:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:56,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014956488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:56,491 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_1' in efdfe7abed14b673b10bbcdae68405c7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:14:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:56,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T11:14:56,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014956657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:56,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014956657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:56,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014956657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:56,745 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/c7cb91d518e6487cb1f1896ae10e868b 2024-11-19T11:14:56,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/2faaa5f5b003431d873b226a402817eb is 50, key is test_row_0/C:col10/1732014895378/Put/seqid=0 2024-11-19T11:14:56,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741998_1174 (size=12151) 2024-11-19T11:14:57,162 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/2faaa5f5b003431d873b226a402817eb 2024-11-19T11:14:57,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014957168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:57,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:57,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:57,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:57,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014957168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:57,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:57,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014957168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:57,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:57,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:57,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:57,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:57,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:57,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/a2119629aae747128c043f4bc6c7cf46 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/a2119629aae747128c043f4bc6c7cf46 2024-11-19T11:14:57,189 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/a2119629aae747128c043f4bc6c7cf46, entries=150, sequenceid=172, filesize=30.4 K 2024-11-19T11:14:57,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/c7cb91d518e6487cb1f1896ae10e868b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/c7cb91d518e6487cb1f1896ae10e868b 2024-11-19T11:14:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,198 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/c7cb91d518e6487cb1f1896ae10e868b, entries=150, sequenceid=172, filesize=11.9 K 2024-11-19T11:14:57,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/2faaa5f5b003431d873b226a402817eb as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/2faaa5f5b003431d873b226a402817eb 2024-11-19T11:14:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,215 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/2faaa5f5b003431d873b226a402817eb, entries=150, sequenceid=172, filesize=11.9 K 2024-11-19T11:14:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,217 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~385.33 KB/394576, currentSize=46.96 KB/48090 for efdfe7abed14b673b10bbcdae68405c7 in 1320ms, sequenceid=172, compaction requested=true 2024-11-19T11:14:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:14:57,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:57,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-19T11:14:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-19T11:14:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-19T11:14:57,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8580 sec 2024-11-19T11:14:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.8670 sec 2024-11-19T11:14:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:57,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efdfe7abed14b673b10bbcdae68405c7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T11:14:58,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=A 2024-11-19T11:14:58,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:58,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=B 2024-11-19T11:14:58,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:58,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=C 2024-11-19T11:14:58,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:58,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:58,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191c8dec15d29d43b0bb9a20efc05e0a20_efdfe7abed14b673b10bbcdae68405c7 is 50, key is test_row_0/A:col10/1732014896027/Put/seqid=0 2024-11-19T11:14:58,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741999_1175 (size=12304) 2024-11-19T11:14:58,226 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,234 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191c8dec15d29d43b0bb9a20efc05e0a20_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191c8dec15d29d43b0bb9a20efc05e0a20_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:58,236 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/8759eaca84584e978994f779ba543104, store: [table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:14:58,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/8759eaca84584e978994f779ba543104 is 175, key is test_row_0/A:col10/1732014896027/Put/seqid=0 2024-11-19T11:14:58,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732014898215, max timestamp=1732014898253Num uniques 859; 2024-11-19T11:14:58,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732014898215, max timestamp=1732014898253Num uniques 859; 2024-11-19T11:14:58,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T11:14:58,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T11:14:58,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732014898215, max timestamp=1732014898253Num uniques 859; 2024-11-19T11:14:58,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T11:14:58,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742000_1176 (size=31105) 2024-11-19T11:14:58,290 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/8759eaca84584e978994f779ba543104 2024-11-19T11:14:58,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/d63077e6febb4ed7b6bc6d19fd431afc is 50, key is test_row_0/B:col10/1732014896027/Put/seqid=0 2024-11-19T11:14:58,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014958311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:58,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:58,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014958322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:58,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:58,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014958323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:58,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742001_1177 (size=12151) 2024-11-19T11:14:58,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/d63077e6febb4ed7b6bc6d19fd431afc 2024-11-19T11:14:58,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/b27fb25e37274738bd0f6ed42f4b144b is 50, key is test_row_0/C:col10/1732014896027/Put/seqid=0 2024-11-19T11:14:58,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742002_1178 (size=12151) 2024-11-19T11:14:58,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/b27fb25e37274738bd0f6ed42f4b144b 2024-11-19T11:14:58,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/8759eaca84584e978994f779ba543104 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/8759eaca84584e978994f779ba543104 2024-11-19T11:14:58,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/8759eaca84584e978994f779ba543104, entries=150, sequenceid=183, filesize=30.4 K 2024-11-19T11:14:58,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/d63077e6febb4ed7b6bc6d19fd431afc as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/d63077e6febb4ed7b6bc6d19fd431afc 2024-11-19T11:14:58,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/d63077e6febb4ed7b6bc6d19fd431afc, entries=150, sequenceid=183, filesize=11.9 K 2024-11-19T11:14:58,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/b27fb25e37274738bd0f6ed42f4b144b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/b27fb25e37274738bd0f6ed42f4b144b 2024-11-19T11:14:58,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/b27fb25e37274738bd0f6ed42f4b144b, entries=150, sequenceid=183, filesize=11.9 K 2024-11-19T11:14:58,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for efdfe7abed14b673b10bbcdae68405c7 in 238ms, sequenceid=183, compaction requested=true 2024-11-19T11:14:58,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:14:58,421 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:14:58,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efdfe7abed14b673b10bbcdae68405c7:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:14:58,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:58,421 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:14:58,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efdfe7abed14b673b10bbcdae68405c7:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:14:58,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:58,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efdfe7abed14b673b10bbcdae68405c7:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:14:58,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:58,422 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133104 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:14:58,422 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): efdfe7abed14b673b10bbcdae68405c7/A is initiating minor compaction (all files) 2024-11-19T11:14:58,422 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efdfe7abed14b673b10bbcdae68405c7/A in TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:58,422 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/7521958460c44954b4d36e316352e710, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/19462fd78f384776bb9bb95e87126cb4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/a2119629aae747128c043f4bc6c7cf46, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/8759eaca84584e978994f779ba543104] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp, totalSize=130.0 K 2024-11-19T11:14:58,422 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:58,422 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. files: [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/7521958460c44954b4d36e316352e710, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/19462fd78f384776bb9bb95e87126cb4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/a2119629aae747128c043f4bc6c7cf46, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/8759eaca84584e978994f779ba543104] 2024-11-19T11:14:58,423 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:14:58,423 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): efdfe7abed14b673b10bbcdae68405c7/B is initiating minor compaction (all files) 2024-11-19T11:14:58,423 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efdfe7abed14b673b10bbcdae68405c7/B in TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:58,423 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/3551ffe615a94c35a88328300766ec92, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/3bcb6fde91f040d198146b6e462f1ead, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/c7cb91d518e6487cb1f1896ae10e868b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/d63077e6febb4ed7b6bc6d19fd431afc] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp, totalSize=47.5 K 2024-11-19T11:14:58,423 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7521958460c44954b4d36e316352e710, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732014894127 2024-11-19T11:14:58,424 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 3551ffe615a94c35a88328300766ec92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732014894127 2024-11-19T11:14:58,424 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19462fd78f384776bb9bb95e87126cb4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732014894289 2024-11-19T11:14:58,424 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bcb6fde91f040d198146b6e462f1ead, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732014894289 2024-11-19T11:14:58,424 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2119629aae747128c043f4bc6c7cf46, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732014895368 2024-11-19T11:14:58,425 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting c7cb91d518e6487cb1f1896ae10e868b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732014895368 2024-11-19T11:14:58,425 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8759eaca84584e978994f779ba543104, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732014896008 2024-11-19T11:14:58,428 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting d63077e6febb4ed7b6bc6d19fd431afc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732014896008 2024-11-19T11:14:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:58,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efdfe7abed14b673b10bbcdae68405c7 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-19T11:14:58,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=A 2024-11-19T11:14:58,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:58,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=B 2024-11-19T11:14:58,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:58,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=C 2024-11-19T11:14:58,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:14:58,449 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:14:58,459 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efdfe7abed14b673b10bbcdae68405c7#B#compaction#153 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:58,459 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/3037e75bae4f4bc790430ed144a5f89c is 50, key is test_row_0/B:col10/1732014896027/Put/seqid=0 2024-11-19T11:14:58,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014958450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:58,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111975261449ad89408893a0f58ae4c79ef5_efdfe7abed14b673b10bbcdae68405c7 is 50, key is test_row_0/A:col10/1732014898290/Put/seqid=0 2024-11-19T11:14:58,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014958462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:58,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014958464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:58,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:58,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:58,473 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119e8c2e277aa5b46ea89120590a9493e90_efdfe7abed14b673b10bbcdae68405c7 store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:14:58,475 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119e8c2e277aa5b46ea89120590a9493e90_efdfe7abed14b673b10bbcdae68405c7, store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:14:58,475 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e8c2e277aa5b46ea89120590a9493e90_efdfe7abed14b673b10bbcdae68405c7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:14:58,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:58,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014958479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:58,484 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_1' in efdfe7abed14b673b10bbcdae68405c7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:14:58,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:58,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742003_1179 (size=12527) 2024-11-19T11:14:58,508 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/3037e75bae4f4bc790430ed144a5f89c as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/3037e75bae4f4bc790430ed144a5f89c 2024-11-19T11:14:58,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742004_1180 (size=17284) 2024-11-19T11:14:58,514 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:58,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742005_1181 (size=4469) 2024-11-19T11:14:58,518 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111975261449ad89408893a0f58ae4c79ef5_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111975261449ad89408893a0f58ae4c79ef5_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:58,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T11:14:58,519 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:14:58,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:58,520 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:14:58,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014958518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:58,520 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efdfe7abed14b673b10bbcdae68405c7/B of efdfe7abed14b673b10bbcdae68405c7 into 3037e75bae4f4bc790430ed144a5f89c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:58,520 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:14:58,520 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., storeName=efdfe7abed14b673b10bbcdae68405c7/B, priority=12, startTime=1732014898421; duration=0sec 2024-11-19T11:14:58,521 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:14:58,521 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efdfe7abed14b673b10bbcdae68405c7:B 2024-11-19T11:14:58,521 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:14:58,520 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_1' in efdfe7abed14b673b10bbcdae68405c7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:14:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:58,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-19T11:14:58,523 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:14:58,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T11:14:58,525 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/a57b7ea1480d487381f4e741cc6313c9, store: [table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:14:58,525 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efdfe7abed14b673b10bbcdae68405c7#A#compaction#152 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:58,525 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:14:58,525 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:14:58,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/a57b7ea1480d487381f4e741cc6313c9 is 175, key is test_row_0/A:col10/1732014898290/Put/seqid=0 2024-11-19T11:14:58,526 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/bfa594b878d1406d86f00bac0d26f5da is 175, key is test_row_0/A:col10/1732014896027/Put/seqid=0 2024-11-19T11:14:58,528 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:14:58,529 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): efdfe7abed14b673b10bbcdae68405c7/C is initiating minor compaction (all files) 2024-11-19T11:14:58,529 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efdfe7abed14b673b10bbcdae68405c7/C in TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:58,529 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/1ebd602cbbba4cf8b363eddd35ddada1, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/4f47ee358bb341df835fa15669b90314, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/2faaa5f5b003431d873b226a402817eb, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/b27fb25e37274738bd0f6ed42f4b144b] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp, totalSize=47.5 K 2024-11-19T11:14:58,529 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ebd602cbbba4cf8b363eddd35ddada1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732014894127 2024-11-19T11:14:58,530 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f47ee358bb341df835fa15669b90314, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732014894289 2024-11-19T11:14:58,530 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 2faaa5f5b003431d873b226a402817eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732014895368 2024-11-19T11:14:58,531 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting b27fb25e37274738bd0f6ed42f4b144b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732014896008 2024-11-19T11:14:58,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742006_1182 (size=48389) 2024-11-19T11:14:58,552 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/a57b7ea1480d487381f4e741cc6313c9 2024-11-19T11:14:58,568 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efdfe7abed14b673b10bbcdae68405c7#C#compaction#155 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:14:58,569 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/67d5d4196a744f5a828342e7f633f7cf is 50, key is test_row_0/C:col10/1732014896027/Put/seqid=0 2024-11-19T11:14:58,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:58,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014958570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:58,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:58,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742007_1183 (size=31481) 2024-11-19T11:14:58,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:58,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014958578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:58,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:58,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:58,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014958578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:58,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:58,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/205066e086e54eb4856213eb2fa65561 is 50, key is test_row_0/B:col10/1732014898290/Put/seqid=0 2024-11-19T11:14:58,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742008_1184 (size=12527) 2024-11-19T11:14:58,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742009_1185 (size=12151) 2024-11-19T11:14:58,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T11:14:58,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T11:14:58,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:58,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:58,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:58,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:58,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:58,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:58,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:58,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014958777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:58,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:58,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014958787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:58,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014958787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:58,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:58,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T11:14:58,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:58,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:58,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:58,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:58,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:58,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T11:14:58,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T11:14:58,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:58,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:58,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:58,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:58,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:58,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:58,990 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/bfa594b878d1406d86f00bac0d26f5da as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/bfa594b878d1406d86f00bac0d26f5da 2024-11-19T11:14:58,997 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efdfe7abed14b673b10bbcdae68405c7/A of efdfe7abed14b673b10bbcdae68405c7 into bfa594b878d1406d86f00bac0d26f5da(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:58,997 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:14:58,997 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., storeName=efdfe7abed14b673b10bbcdae68405c7/A, priority=12, startTime=1732014898420; duration=0sec 2024-11-19T11:14:58,997 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:58,997 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efdfe7abed14b673b10bbcdae68405c7:A 2024-11-19T11:14:59,014 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/67d5d4196a744f5a828342e7f633f7cf as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/67d5d4196a744f5a828342e7f633f7cf 2024-11-19T11:14:59,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/205066e086e54eb4856213eb2fa65561 2024-11-19T11:14:59,019 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efdfe7abed14b673b10bbcdae68405c7/C of efdfe7abed14b673b10bbcdae68405c7 into 67d5d4196a744f5a828342e7f633f7cf(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:14:59,019 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:14:59,019 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., storeName=efdfe7abed14b673b10bbcdae68405c7/C, priority=12, startTime=1732014898421; duration=0sec 2024-11-19T11:14:59,019 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:14:59,019 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efdfe7abed14b673b10bbcdae68405c7:C 2024-11-19T11:14:59,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/af0764ae21fc41248843e96093e6fb13 is 50, key is test_row_0/C:col10/1732014898290/Put/seqid=0 2024-11-19T11:14:59,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742010_1186 (size=12151) 2024-11-19T11:14:59,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/af0764ae21fc41248843e96093e6fb13 2024-11-19T11:14:59,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/a57b7ea1480d487381f4e741cc6313c9 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/a57b7ea1480d487381f4e741cc6313c9 2024-11-19T11:14:59,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/a57b7ea1480d487381f4e741cc6313c9, entries=250, sequenceid=220, filesize=47.3 K 2024-11-19T11:14:59,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/205066e086e54eb4856213eb2fa65561 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/205066e086e54eb4856213eb2fa65561 2024-11-19T11:14:59,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/205066e086e54eb4856213eb2fa65561, entries=150, sequenceid=220, filesize=11.9 K 2024-11-19T11:14:59,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/af0764ae21fc41248843e96093e6fb13 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/af0764ae21fc41248843e96093e6fb13 2024-11-19T11:14:59,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/af0764ae21fc41248843e96093e6fb13, entries=150, sequenceid=220, filesize=11.9 K 2024-11-19T11:14:59,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for efdfe7abed14b673b10bbcdae68405c7 in 613ms, sequenceid=220, compaction requested=false 2024-11-19T11:14:59,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:14:59,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efdfe7abed14b673b10bbcdae68405c7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T11:14:59,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:59,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=A 2024-11-19T11:14:59,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:59,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=B 2024-11-19T11:14:59,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:59,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=C 2024-11-19T11:14:59,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:14:59,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bee4abe4285e4ddeb936264f0d761604_efdfe7abed14b673b10bbcdae68405c7 is 50, key is test_row_0/A:col10/1732014898447/Put/seqid=0 2024-11-19T11:14:59,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742011_1187 (size=12304) 2024-11-19T11:14:59,116 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:14:59,122 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bee4abe4285e4ddeb936264f0d761604_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bee4abe4285e4ddeb936264f0d761604_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:14:59,123 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/e1a643230fa6436eb38c044eb140b5fd, store: [table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:14:59,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/e1a643230fa6436eb38c044eb140b5fd is 175, key is test_row_0/A:col10/1732014898447/Put/seqid=0 2024-11-19T11:14:59,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742012_1188 (size=31105) 2024-11-19T11:14:59,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T11:14:59,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:59,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:59,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:59,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:59,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:59,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:59,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014899118, max timestamp=1732014899137Num uniques 859; 2024-11-19T11:14:59,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:14:59,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014899118, max timestamp=1732014899137Num uniques 859; 2024-11-19T11:14:59,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:14:59,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014899118, max timestamp=1732014899137Num uniques 859; 2024-11-19T11:14:59,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:14:59,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T11:14:59,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:59,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014959178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:59,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732014899165, max timestamp=1732014899179Num uniques 1018; 2024-11-19T11:14:59,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732014899165, max timestamp=1732014899179Num uniques 1018; 2024-11-19T11:14:59,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-19T11:14:59,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732014899165, max timestamp=1732014899179Num uniques 1018; 2024-11-19T11:14:59,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-19T11:14:59,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:59,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-19T11:14:59,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014959214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:59,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:59,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014959215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:59,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:59,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:59,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014959287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:59,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T11:14:59,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:59,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:59,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:59,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:59,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:59,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014959327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:59,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014959327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:59,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T11:14:59,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:59,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:59,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:59,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:59,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:59,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:59,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014959498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:59,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:59,527 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/e1a643230fa6436eb38c044eb140b5fd 2024-11-19T11:14:59,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/d8c22d2ae3714604b554df98bb918954 is 50, key is test_row_0/B:col10/1732014898447/Put/seqid=0 2024-11-19T11:14:59,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742013_1189 (size=12151) 2024-11-19T11:14:59,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014959537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:59,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014959537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:59,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T11:14:59,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:59,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:59,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:59,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:59,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:59,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:59,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T11:14:59,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T11:14:59,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:59,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:59,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:59,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:59,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:59,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:59,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:59,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014959808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:59,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:59,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:59,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014959847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:59,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:14:59,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014959847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:14:59,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:59,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:14:59,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:14:59,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T11:14:59,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:59,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:14:59,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:14:59,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:59,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:14:59,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/d8c22d2ae3714604b554df98bb918954 2024-11-19T11:14:59,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/757e26c880b54ae6bcae475a7e9eab8d is 50, key is test_row_0/C:col10/1732014898447/Put/seqid=0 2024-11-19T11:14:59,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742014_1190 (size=12151) 2024-11-19T11:14:59,962 INFO [master/16c88966c0af:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-19T11:14:59,962 INFO [master/16c88966c0af:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-19T11:15:00,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T11:15:00,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:00,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:15:00,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:00,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:00,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:00,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:00,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T11:15:00,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:00,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:15:00,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:00,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:00,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:00,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014960327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:00,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/757e26c880b54ae6bcae475a7e9eab8d 2024-11-19T11:15:00,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/e1a643230fa6436eb38c044eb140b5fd as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/e1a643230fa6436eb38c044eb140b5fd 2024-11-19T11:15:00,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:00,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014960358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:00,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:00,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:00,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:00,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:00,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014960358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:00,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:00,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:00,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:00,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T11:15:00,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/e1a643230fa6436eb38c044eb140b5fd, entries=150, sequenceid=233, filesize=30.4 K 2024-11-19T11:15:00,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:00,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:15:00,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:00,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:00,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:00,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/d8c22d2ae3714604b554df98bb918954 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/d8c22d2ae3714604b554df98bb918954 2024-11-19T11:15:00,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/d8c22d2ae3714604b554df98bb918954, entries=150, sequenceid=233, filesize=11.9 K 2024-11-19T11:15:00,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/757e26c880b54ae6bcae475a7e9eab8d as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/757e26c880b54ae6bcae475a7e9eab8d 2024-11-19T11:15:00,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/757e26c880b54ae6bcae475a7e9eab8d, entries=150, sequenceid=233, filesize=11.9 K 2024-11-19T11:15:00,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for efdfe7abed14b673b10bbcdae68405c7 in 1285ms, sequenceid=233, compaction requested=true 2024-11-19T11:15:00,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:15:00,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efdfe7abed14b673b10bbcdae68405c7:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:15:00,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:00,379 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:00,379 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:00,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efdfe7abed14b673b10bbcdae68405c7:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:15:00,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:00,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efdfe7abed14b673b10bbcdae68405c7:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:15:00,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:00,380 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:00,380 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): efdfe7abed14b673b10bbcdae68405c7/B is initiating minor compaction (all files) 2024-11-19T11:15:00,380 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efdfe7abed14b673b10bbcdae68405c7/B in TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:00,381 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/3037e75bae4f4bc790430ed144a5f89c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/205066e086e54eb4856213eb2fa65561, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/d8c22d2ae3714604b554df98bb918954] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp, totalSize=36.0 K 2024-11-19T11:15:00,381 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110975 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:00,381 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): efdfe7abed14b673b10bbcdae68405c7/A is initiating minor compaction (all files) 2024-11-19T11:15:00,381 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efdfe7abed14b673b10bbcdae68405c7/A in TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:00,381 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/bfa594b878d1406d86f00bac0d26f5da, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/a57b7ea1480d487381f4e741cc6313c9, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/e1a643230fa6436eb38c044eb140b5fd] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp, totalSize=108.4 K 2024-11-19T11:15:00,381 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:00,381 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. files: [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/bfa594b878d1406d86f00bac0d26f5da, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/a57b7ea1480d487381f4e741cc6313c9, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/e1a643230fa6436eb38c044eb140b5fd] 2024-11-19T11:15:00,383 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 3037e75bae4f4bc790430ed144a5f89c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732014896008 2024-11-19T11:15:00,383 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfa594b878d1406d86f00bac0d26f5da, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732014896008 2024-11-19T11:15:00,383 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 205066e086e54eb4856213eb2fa65561, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732014898290 2024-11-19T11:15:00,384 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting a57b7ea1480d487381f4e741cc6313c9, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732014898290 2024-11-19T11:15:00,384 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting d8c22d2ae3714604b554df98bb918954, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732014898445 2024-11-19T11:15:00,384 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1a643230fa6436eb38c044eb140b5fd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732014898445 2024-11-19T11:15:00,391 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:15:00,393 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efdfe7abed14b673b10bbcdae68405c7#B#compaction#162 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:00,394 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/f609e3c13ca3414795244b26dea0b418 is 50, key is test_row_0/B:col10/1732014898447/Put/seqid=0 2024-11-19T11:15:00,396 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411198d28c0f06e4b4c7a92b82af6ae2c7cd7_efdfe7abed14b673b10bbcdae68405c7 store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:15:00,398 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411198d28c0f06e4b4c7a92b82af6ae2c7cd7_efdfe7abed14b673b10bbcdae68405c7, store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:15:00,399 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198d28c0f06e4b4c7a92b82af6ae2c7cd7_efdfe7abed14b673b10bbcdae68405c7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:15:00,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742015_1191 (size=12629) 2024-11-19T11:15:00,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742016_1192 (size=4469) 2024-11-19T11:15:00,420 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/f609e3c13ca3414795244b26dea0b418 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/f609e3c13ca3414795244b26dea0b418 2024-11-19T11:15:00,426 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efdfe7abed14b673b10bbcdae68405c7/B of efdfe7abed14b673b10bbcdae68405c7 into f609e3c13ca3414795244b26dea0b418(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:00,426 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:15:00,426 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., storeName=efdfe7abed14b673b10bbcdae68405c7/B, priority=13, startTime=1732014900379; duration=0sec 2024-11-19T11:15:00,426 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:00,426 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efdfe7abed14b673b10bbcdae68405c7:B 2024-11-19T11:15:00,426 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:00,427 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:00,427 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): efdfe7abed14b673b10bbcdae68405c7/C is initiating minor compaction (all files) 2024-11-19T11:15:00,427 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efdfe7abed14b673b10bbcdae68405c7/C in TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:00,428 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/67d5d4196a744f5a828342e7f633f7cf, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/af0764ae21fc41248843e96093e6fb13, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/757e26c880b54ae6bcae475a7e9eab8d] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp, totalSize=36.0 K 2024-11-19T11:15:00,428 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 67d5d4196a744f5a828342e7f633f7cf, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732014896008 2024-11-19T11:15:00,428 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting af0764ae21fc41248843e96093e6fb13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732014898290 2024-11-19T11:15:00,428 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 757e26c880b54ae6bcae475a7e9eab8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732014898445 2024-11-19T11:15:00,434 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efdfe7abed14b673b10bbcdae68405c7#C#compaction#163 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:00,435 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/4fa1a763b6e64567afa93fdf6d8243eb is 50, key is test_row_0/C:col10/1732014898447/Put/seqid=0 2024-11-19T11:15:00,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742017_1193 (size=12629) 2024-11-19T11:15:00,446 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/4fa1a763b6e64567afa93fdf6d8243eb as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/4fa1a763b6e64567afa93fdf6d8243eb 2024-11-19T11:15:00,451 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efdfe7abed14b673b10bbcdae68405c7/C of efdfe7abed14b673b10bbcdae68405c7 into 4fa1a763b6e64567afa93fdf6d8243eb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:00,451 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:15:00,451 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., storeName=efdfe7abed14b673b10bbcdae68405c7/C, priority=13, startTime=1732014900379; duration=0sec 2024-11-19T11:15:00,451 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:00,451 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efdfe7abed14b673b10bbcdae68405c7:C 2024-11-19T11:15:00,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T11:15:00,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:00,519 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing efdfe7abed14b673b10bbcdae68405c7 3/3 column families, dataSize=134.18 KB heapSize=374.77 KB 2024-11-19T11:15:00,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=A 2024-11-19T11:15:00,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:00,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=B 2024-11-19T11:15:00,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:00,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=C 2024-11-19T11:15:00,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:00,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bc662b9ac272468cbc36e0b16797186e_efdfe7abed14b673b10bbcdae68405c7 is 50, key is test_row_0/A:col10/1732014899214/Put/seqid=0 2024-11-19T11:15:00,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742018_1194 (size=12454) 2024-11-19T11:15:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T11:15:00,809 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efdfe7abed14b673b10bbcdae68405c7#A#compaction#161 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:00,810 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/9e6614e80b9741f4af5090e3f9eaf29b is 175, key is test_row_0/A:col10/1732014898447/Put/seqid=0 2024-11-19T11:15:00,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742019_1195 (size=31583) 2024-11-19T11:15:00,822 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/9e6614e80b9741f4af5090e3f9eaf29b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/9e6614e80b9741f4af5090e3f9eaf29b 2024-11-19T11:15:00,827 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efdfe7abed14b673b10bbcdae68405c7/A of efdfe7abed14b673b10bbcdae68405c7 into 9e6614e80b9741f4af5090e3f9eaf29b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:00,827 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:15:00,827 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., storeName=efdfe7abed14b673b10bbcdae68405c7/A, priority=13, startTime=1732014900379; duration=0sec 2024-11-19T11:15:00,828 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:00,828 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efdfe7abed14b673b10bbcdae68405c7:A 2024-11-19T11:15:00,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:00,938 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bc662b9ac272468cbc36e0b16797186e_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bc662b9ac272468cbc36e0b16797186e_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:00,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/99f7f6d2497c45c4aca5cf4bc8ad3283, store: [table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:15:00,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/99f7f6d2497c45c4aca5cf4bc8ad3283 is 175, key is test_row_0/A:col10/1732014899214/Put/seqid=0 2024-11-19T11:15:00,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742020_1196 (size=31255) 2024-11-19T11:15:01,344 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/99f7f6d2497c45c4aca5cf4bc8ad3283 2024-11-19T11:15:01,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/e92b27da74414dbd8fab16af85c27009 is 50, key is test_row_0/B:col10/1732014899214/Put/seqid=0 2024-11-19T11:15:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:01,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:15:01,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742021_1197 (size=12301) 2024-11-19T11:15:01,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:01,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:01,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014961370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:01,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014961371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:01,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:01,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014961372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:01,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:01,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:01,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:01,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014961478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:01,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014961478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:01,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:01,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:01,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:01,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:01,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:01,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:01,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014961487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:01,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:01,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:01,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:01,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:01,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014961688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:01,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:01,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014961688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:01,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014961697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:01,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:01,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:01,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:01,754 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/e92b27da74414dbd8fab16af85c27009 2024-11-19T11:15:01,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/e3d97da94b8c4c4caeb4e850d948eb2d is 50, key is test_row_0/C:col10/1732014899214/Put/seqid=0 2024-11-19T11:15:01,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742022_1198 (size=12301) 2024-11-19T11:15:02,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014961997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:02,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014961998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:02,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:02,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014962008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:02,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:02,165 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/e3d97da94b8c4c4caeb4e850d948eb2d 2024-11-19T11:15:02,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/99f7f6d2497c45c4aca5cf4bc8ad3283 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/99f7f6d2497c45c4aca5cf4bc8ad3283 2024-11-19T11:15:02,173 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/99f7f6d2497c45c4aca5cf4bc8ad3283, entries=150, sequenceid=287, filesize=30.5 K 2024-11-19T11:15:02,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/e92b27da74414dbd8fab16af85c27009 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/e92b27da74414dbd8fab16af85c27009 2024-11-19T11:15:02,178 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/e92b27da74414dbd8fab16af85c27009, entries=150, sequenceid=287, filesize=12.0 K 2024-11-19T11:15:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/e3d97da94b8c4c4caeb4e850d948eb2d as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/e3d97da94b8c4c4caeb4e850d948eb2d 2024-11-19T11:15:02,183 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/e3d97da94b8c4c4caeb4e850d948eb2d, entries=150, sequenceid=287, filesize=12.0 K 2024-11-19T11:15:02,184 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~374.72 KB/383712, currentSize=46.96 KB/48090 for efdfe7abed14b673b10bbcdae68405c7 in 1665ms, sequenceid=287, compaction requested=false 2024-11-19T11:15:02,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:15:02,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:02,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-19T11:15:02,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-19T11:15:02,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-19T11:15:02,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6600 sec 2024-11-19T11:15:02,189 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 3.6670 sec 2024-11-19T11:15:02,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:02,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efdfe7abed14b673b10bbcdae68405c7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T11:15:02,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=A 2024-11-19T11:15:02,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:02,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=B 2024-11-19T11:15:02,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:02,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=C 2024-11-19T11:15:02,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:02,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192d59c435d2944f13bc21c14047df8001_efdfe7abed14b673b10bbcdae68405c7 is 50, key is test_row_0/A:col10/1732014901369/Put/seqid=0 2024-11-19T11:15:02,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742023_1199 (size=14994) 2024-11-19T11:15:02,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:02,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014962527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:02,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:02,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014962528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:02,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:02,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:02,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732014902517, max timestamp=1732014902530Num uniques 859; 2024-11-19T11:15:02,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:02,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T11:15:02,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014962529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:02,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732014902517, max timestamp=1732014902530Num uniques 859; 2024-11-19T11:15:02,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732014902517, max timestamp=1732014902530Num uniques 859; 2024-11-19T11:15:02,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T11:15:02,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T11:15:02,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:02,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:02,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014962584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:02,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:02,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014962585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:02,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014962637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:02,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:02,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014962637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014962638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T11:15:02,687 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:15:02,688 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:15:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-19T11:15:02,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T11:15:02,689 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:15:02,690 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:15:02,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:15:02,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014962687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:02,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014962687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:02,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T11:15:02,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T11:15:02,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:02,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:15:02,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:02,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:02,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:02,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:02,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:02,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014962848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:02,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:02,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014962848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:02,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:02,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:02,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014962848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:02,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:02,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:02,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014962897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:02,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:02,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014962898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:02,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:02,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:02,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:02,919 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:02,926 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192d59c435d2944f13bc21c14047df8001_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192d59c435d2944f13bc21c14047df8001_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:02,927 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/473c9613f4854ef0a9d5afcae0cb39bf, store: [table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:15:02,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/473c9613f4854ef0a9d5afcae0cb39bf is 175, key is test_row_0/A:col10/1732014901369/Put/seqid=0 2024-11-19T11:15:02,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742024_1200 (size=39949) 2024-11-19T11:15:02,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T11:15:02,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:02,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:15:02,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:02,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:02,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:03,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T11:15:03,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T11:15:03,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:03,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:15:03,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:03,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:03,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:03,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:03,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:03,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014963158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:03,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:03,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:03,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014963158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:03,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:03,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:03,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:03,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:03,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014963158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:03,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:03,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:03,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:03,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:03,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:03,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014963207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:03,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014963207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:03,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T11:15:03,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:03,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:15:03,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:03,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:03,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T11:15:03,334 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/473c9613f4854ef0a9d5afcae0cb39bf 2024-11-19T11:15:03,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/6723203fffdf4d328ebbd22bb25f5d87 is 50, key is test_row_0/B:col10/1732014901369/Put/seqid=0 2024-11-19T11:15:03,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742025_1201 (size=12301) 2024-11-19T11:15:03,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T11:15:03,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:03,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:15:03,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:03,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:03,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:03,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T11:15:03,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:03,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:15:03,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:03,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:03,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:03,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014963667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:03,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014963667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:03,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014963667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:03,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014963718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:03,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:03,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:03,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:03,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014963727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:03,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:03,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:03,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:03,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/6723203fffdf4d328ebbd22bb25f5d87 2024-11-19T11:15:03,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/53e6c63706194fcaabd315fc228c3291 is 50, key is test_row_0/C:col10/1732014901369/Put/seqid=0 2024-11-19T11:15:03,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742026_1202 (size=12301) 2024-11-19T11:15:03,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T11:15:03,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:03,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:15:03,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:03,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:03,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:03,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/53e6c63706194fcaabd315fc228c3291 2024-11-19T11:15:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:03,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/473c9613f4854ef0a9d5afcae0cb39bf as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/473c9613f4854ef0a9d5afcae0cb39bf 2024-11-19T11:15:03,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/473c9613f4854ef0a9d5afcae0cb39bf, entries=200, sequenceid=299, filesize=39.0 K 2024-11-19T11:15:03,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/6723203fffdf4d328ebbd22bb25f5d87 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/6723203fffdf4d328ebbd22bb25f5d87 2024-11-19T11:15:03,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/6723203fffdf4d328ebbd22bb25f5d87, entries=150, sequenceid=299, filesize=12.0 K 2024-11-19T11:15:03,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/53e6c63706194fcaabd315fc228c3291 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/53e6c63706194fcaabd315fc228c3291 2024-11-19T11:15:03,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/53e6c63706194fcaabd315fc228c3291, entries=150, sequenceid=299, filesize=12.0 K 2024-11-19T11:15:03,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for efdfe7abed14b673b10bbcdae68405c7 in 1278ms, sequenceid=299, compaction requested=true 2024-11-19T11:15:03,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:15:03,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efdfe7abed14b673b10bbcdae68405c7:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:15:03,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:03,778 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:03,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efdfe7abed14b673b10bbcdae68405c7:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:15:03,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:03,778 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:03,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efdfe7abed14b673b10bbcdae68405c7:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:15:03,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:03,779 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:03,779 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): efdfe7abed14b673b10bbcdae68405c7/B is initiating minor compaction (all files) 2024-11-19T11:15:03,779 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efdfe7abed14b673b10bbcdae68405c7/B in TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:03,780 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/f609e3c13ca3414795244b26dea0b418, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/e92b27da74414dbd8fab16af85c27009, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/6723203fffdf4d328ebbd22bb25f5d87] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp, totalSize=36.4 K 2024-11-19T11:15:03,780 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102787 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:03,780 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): efdfe7abed14b673b10bbcdae68405c7/A is initiating minor compaction (all files) 2024-11-19T11:15:03,780 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efdfe7abed14b673b10bbcdae68405c7/A in TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:03,780 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/9e6614e80b9741f4af5090e3f9eaf29b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/99f7f6d2497c45c4aca5cf4bc8ad3283, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/473c9613f4854ef0a9d5afcae0cb39bf] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp, totalSize=100.4 K 2024-11-19T11:15:03,780 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:03,780 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. files: [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/9e6614e80b9741f4af5090e3f9eaf29b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/99f7f6d2497c45c4aca5cf4bc8ad3283, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/473c9613f4854ef0a9d5afcae0cb39bf] 2024-11-19T11:15:03,781 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting f609e3c13ca3414795244b26dea0b418, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732014898445 2024-11-19T11:15:03,781 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e6614e80b9741f4af5090e3f9eaf29b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732014898445 2024-11-19T11:15:03,781 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 99f7f6d2497c45c4aca5cf4bc8ad3283, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732014899199 2024-11-19T11:15:03,781 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting e92b27da74414dbd8fab16af85c27009, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732014899199 2024-11-19T11:15:03,781 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 473c9613f4854ef0a9d5afcae0cb39bf, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732014901359 2024-11-19T11:15:03,781 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6723203fffdf4d328ebbd22bb25f5d87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732014901369 2024-11-19T11:15:03,788 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:15:03,790 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efdfe7abed14b673b10bbcdae68405c7#B#compaction#170 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:03,791 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/4fc6164f0ada44629978932b6eaa400d is 50, key is test_row_0/B:col10/1732014901369/Put/seqid=0 2024-11-19T11:15:03,793 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111996fb17117e554bae8de5c5c519e8fbad_efdfe7abed14b673b10bbcdae68405c7 store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:15:03,795 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111996fb17117e554bae8de5c5c519e8fbad_efdfe7abed14b673b10bbcdae68405c7, store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:15:03,795 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111996fb17117e554bae8de5c5c519e8fbad_efdfe7abed14b673b10bbcdae68405c7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:15:03,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742027_1203 (size=12881) 2024-11-19T11:15:03,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742028_1204 (size=4469) 2024-11-19T11:15:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T11:15:03,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T11:15:03,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:03,913 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing efdfe7abed14b673b10bbcdae68405c7 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-19T11:15:03,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=A 2024-11-19T11:15:03,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:03,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=B 2024-11-19T11:15:03,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:03,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=C 2024-11-19T11:15:03,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:03,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199e43b80408e9420891c7b1daadb84ccb_efdfe7abed14b673b10bbcdae68405c7 is 50, key is test_row_0/A:col10/1732014902582/Put/seqid=0 2024-11-19T11:15:03,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742029_1205 (size=12454) 2024-11-19T11:15:04,213 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/4fc6164f0ada44629978932b6eaa400d as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/4fc6164f0ada44629978932b6eaa400d 2024-11-19T11:15:04,217 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efdfe7abed14b673b10bbcdae68405c7#A#compaction#171 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:04,217 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/1a6702ff54e7434cb6bf1b0255ffcb73 is 175, key is test_row_0/A:col10/1732014901369/Put/seqid=0 2024-11-19T11:15:04,217 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efdfe7abed14b673b10bbcdae68405c7/B of efdfe7abed14b673b10bbcdae68405c7 into 4fc6164f0ada44629978932b6eaa400d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:04,217 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:15:04,217 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., storeName=efdfe7abed14b673b10bbcdae68405c7/B, priority=13, startTime=1732014903778; duration=0sec 2024-11-19T11:15:04,217 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:04,217 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efdfe7abed14b673b10bbcdae68405c7:B 2024-11-19T11:15:04,217 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:04,219 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:04,219 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): efdfe7abed14b673b10bbcdae68405c7/C is initiating minor compaction (all files) 2024-11-19T11:15:04,219 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efdfe7abed14b673b10bbcdae68405c7/C in TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:04,219 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/4fa1a763b6e64567afa93fdf6d8243eb, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/e3d97da94b8c4c4caeb4e850d948eb2d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/53e6c63706194fcaabd315fc228c3291] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp, totalSize=36.4 K 2024-11-19T11:15:04,219 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fa1a763b6e64567afa93fdf6d8243eb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732014898445 2024-11-19T11:15:04,219 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3d97da94b8c4c4caeb4e850d948eb2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732014899199 2024-11-19T11:15:04,220 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53e6c63706194fcaabd315fc228c3291, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732014901369 2024-11-19T11:15:04,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742030_1206 (size=31835) 2024-11-19T11:15:04,228 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efdfe7abed14b673b10bbcdae68405c7#C#compaction#173 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:04,229 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/42f5791e139c42528cf3d7477a966c7d is 50, key is test_row_0/C:col10/1732014901369/Put/seqid=0 2024-11-19T11:15:04,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742031_1207 (size=12881) 2024-11-19T11:15:04,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:04,326 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199e43b80408e9420891c7b1daadb84ccb_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199e43b80408e9420891c7b1daadb84ccb_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:04,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/443d23d1396a4c31afe257f8867381f4, store: [table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:15:04,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/443d23d1396a4c31afe257f8867381f4 is 175, key is test_row_0/A:col10/1732014902582/Put/seqid=0 2024-11-19T11:15:04,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742032_1208 (size=31255) 2024-11-19T11:15:04,626 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/1a6702ff54e7434cb6bf1b0255ffcb73 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/1a6702ff54e7434cb6bf1b0255ffcb73 2024-11-19T11:15:04,630 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efdfe7abed14b673b10bbcdae68405c7/A of efdfe7abed14b673b10bbcdae68405c7 into 1a6702ff54e7434cb6bf1b0255ffcb73(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:04,630 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:15:04,630 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., storeName=efdfe7abed14b673b10bbcdae68405c7/A, priority=13, startTime=1732014903778; duration=0sec 2024-11-19T11:15:04,631 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:04,631 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efdfe7abed14b673b10bbcdae68405c7:A 2024-11-19T11:15:04,636 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/42f5791e139c42528cf3d7477a966c7d as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/42f5791e139c42528cf3d7477a966c7d 2024-11-19T11:15:04,640 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efdfe7abed14b673b10bbcdae68405c7/C of efdfe7abed14b673b10bbcdae68405c7 into 42f5791e139c42528cf3d7477a966c7d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:04,640 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:15:04,640 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., storeName=efdfe7abed14b673b10bbcdae68405c7/C, priority=13, startTime=1732014903778; duration=0sec 2024-11-19T11:15:04,641 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:04,641 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efdfe7abed14b673b10bbcdae68405c7:C 2024-11-19T11:15:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:04,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:15:04,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:04,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014964691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:04,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:04,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:04,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:04,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:04,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014964692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:04,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:04,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:04,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:04,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:04,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014964693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:04,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:04,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:04,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:04,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:04,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014964728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:04,729 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_1' in efdfe7abed14b673b10bbcdae68405c7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2145 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:15:04,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:04,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:04,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:04,733 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/443d23d1396a4c31afe257f8867381f4 2024-11-19T11:15:04,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/823d24272a854321aeac12a25292f731 is 50, key is test_row_0/B:col10/1732014902582/Put/seqid=0 2024-11-19T11:15:04,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742033_1209 (size=12301) 2024-11-19T11:15:04,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014964747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:04,748 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_2' in efdfe7abed14b673b10bbcdae68405c7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:15:04,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:04,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:04,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:04,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:04,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014964797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:04,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:04,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014964798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:04,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:04,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014964807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:04,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:04,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:04,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T11:15:05,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:05,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014965008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:05,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014965008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:05,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:05,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014965017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:05,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:05,146 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/823d24272a854321aeac12a25292f731 2024-11-19T11:15:05,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/fd885cb6345f467cbf3e721ef668b588 is 50, key is test_row_0/C:col10/1732014902582/Put/seqid=0 2024-11-19T11:15:05,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742034_1210 (size=12301) 2024-11-19T11:15:05,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014965317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:05,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014965317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:05,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:05,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014965327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:05,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:05,557 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/fd885cb6345f467cbf3e721ef668b588 2024-11-19T11:15:05,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/443d23d1396a4c31afe257f8867381f4 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/443d23d1396a4c31afe257f8867381f4 2024-11-19T11:15:05,565 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/443d23d1396a4c31afe257f8867381f4, entries=150, sequenceid=333, filesize=30.5 K 2024-11-19T11:15:05,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/823d24272a854321aeac12a25292f731 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/823d24272a854321aeac12a25292f731 2024-11-19T11:15:05,570 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/823d24272a854321aeac12a25292f731, entries=150, sequenceid=333, filesize=12.0 K 2024-11-19T11:15:05,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/fd885cb6345f467cbf3e721ef668b588 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/fd885cb6345f467cbf3e721ef668b588 2024-11-19T11:15:05,575 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/fd885cb6345f467cbf3e721ef668b588, entries=150, sequenceid=333, filesize=12.0 K 2024-11-19T11:15:05,576 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for efdfe7abed14b673b10bbcdae68405c7 in 1664ms, sequenceid=333, compaction requested=false 2024-11-19T11:15:05,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:15:05,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:05,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-19T11:15:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-19T11:15:05,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-19T11:15:05,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8870 sec 2024-11-19T11:15:05,581 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.8910 sec 2024-11-19T11:15:05,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:05,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efdfe7abed14b673b10bbcdae68405c7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T11:15:05,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=A 2024-11-19T11:15:05,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:05,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=B 2024-11-19T11:15:05,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:05,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=C 2024-11-19T11:15:05,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:05,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196f66de225d4b4c4d902a9464ffe47183_efdfe7abed14b673b10bbcdae68405c7 is 50, key is test_row_0/A:col10/1732014904692/Put/seqid=0 2024-11-19T11:15:05,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742035_1211 (size=17534) 2024-11-19T11:15:05,845 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:05,849 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196f66de225d4b4c4d902a9464ffe47183_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196f66de225d4b4c4d902a9464ffe47183_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:05,850 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/e1b292d9077c4ec18edbfc71b3f01987, store: [table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:15:05,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/e1b292d9077c4ec18edbfc71b3f01987 is 175, key is test_row_0/A:col10/1732014904692/Put/seqid=0 2024-11-19T11:15:05,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742036_1212 (size=48639) 2024-11-19T11:15:05,858 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/e1b292d9077c4ec18edbfc71b3f01987 2024-11-19T11:15:05,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014965856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:05,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:05,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014965857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:05,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:05,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:05,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014965861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:05,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/f400a7c0c5b54f0eb4b467f900501063 is 50, key is test_row_0/B:col10/1732014904692/Put/seqid=0 2024-11-19T11:15:05,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:05,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742037_1213 (size=12301) 2024-11-19T11:15:05,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014965967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:05,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014965967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:05,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:05,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014965967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:05,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:05,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:06,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014966178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014966178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014966178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:06,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/f400a7c0c5b54f0eb4b467f900501063 2024-11-19T11:15:06,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/039a1c8f6d404e0aa756e7046567d593 is 50, key is test_row_0/C:col10/1732014904692/Put/seqid=0 2024-11-19T11:15:06,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742038_1214 (size=12301) 2024-11-19T11:15:06,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/039a1c8f6d404e0aa756e7046567d593 2024-11-19T11:15:06,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/e1b292d9077c4ec18edbfc71b3f01987 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/e1b292d9077c4ec18edbfc71b3f01987 2024-11-19T11:15:06,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/e1b292d9077c4ec18edbfc71b3f01987, entries=250, sequenceid=348, filesize=47.5 K 2024-11-19T11:15:06,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/f400a7c0c5b54f0eb4b467f900501063 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/f400a7c0c5b54f0eb4b467f900501063 2024-11-19T11:15:06,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/f400a7c0c5b54f0eb4b467f900501063, entries=150, sequenceid=348, filesize=12.0 K 2024-11-19T11:15:06,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/039a1c8f6d404e0aa756e7046567d593 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/039a1c8f6d404e0aa756e7046567d593 2024-11-19T11:15:06,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/039a1c8f6d404e0aa756e7046567d593, entries=150, sequenceid=348, filesize=12.0 K 2024-11-19T11:15:06,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for efdfe7abed14b673b10bbcdae68405c7 in 481ms, sequenceid=348, compaction requested=true 2024-11-19T11:15:06,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:15:06,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efdfe7abed14b673b10bbcdae68405c7:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:15:06,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:06,310 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:06,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efdfe7abed14b673b10bbcdae68405c7:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:15:06,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:06,311 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:06,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efdfe7abed14b673b10bbcdae68405c7:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:15:06,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:06,316 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111729 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:06,316 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): efdfe7abed14b673b10bbcdae68405c7/A is initiating minor compaction (all files) 2024-11-19T11:15:06,316 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efdfe7abed14b673b10bbcdae68405c7/A in TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:06,316 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/1a6702ff54e7434cb6bf1b0255ffcb73, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/443d23d1396a4c31afe257f8867381f4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/e1b292d9077c4ec18edbfc71b3f01987] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp, totalSize=109.1 K 2024-11-19T11:15:06,316 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:06,316 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. files: [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/1a6702ff54e7434cb6bf1b0255ffcb73, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/443d23d1396a4c31afe257f8867381f4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/e1b292d9077c4ec18edbfc71b3f01987] 2024-11-19T11:15:06,317 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:06,317 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): efdfe7abed14b673b10bbcdae68405c7/B is initiating minor compaction (all files) 2024-11-19T11:15:06,317 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efdfe7abed14b673b10bbcdae68405c7/B in TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:06,317 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/4fc6164f0ada44629978932b6eaa400d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/823d24272a854321aeac12a25292f731, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/f400a7c0c5b54f0eb4b467f900501063] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp, totalSize=36.6 K 2024-11-19T11:15:06,318 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fc6164f0ada44629978932b6eaa400d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732014901369 2024-11-19T11:15:06,318 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a6702ff54e7434cb6bf1b0255ffcb73, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732014901369 2024-11-19T11:15:06,318 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 823d24272a854321aeac12a25292f731, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732014902566 2024-11-19T11:15:06,318 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 443d23d1396a4c31afe257f8867381f4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732014902566 2024-11-19T11:15:06,318 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting f400a7c0c5b54f0eb4b467f900501063, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732014904692 2024-11-19T11:15:06,318 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1b292d9077c4ec18edbfc71b3f01987, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732014904688 2024-11-19T11:15:06,325 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:15:06,327 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efdfe7abed14b673b10bbcdae68405c7#B#compaction#180 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:06,328 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/7ba73e0526444f51a5b27132dfcff42f is 50, key is test_row_0/B:col10/1732014904692/Put/seqid=0 2024-11-19T11:15:06,330 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411193d76bbefa1db43c18da919724df37e6a_efdfe7abed14b673b10bbcdae68405c7 store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:15:06,331 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411193d76bbefa1db43c18da919724df37e6a_efdfe7abed14b673b10bbcdae68405c7, store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:15:06,332 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193d76bbefa1db43c18da919724df37e6a_efdfe7abed14b673b10bbcdae68405c7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:15:06,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742039_1215 (size=12983) 2024-11-19T11:15:06,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742040_1216 (size=4469) 2024-11-19T11:15:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:06,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014905852, max timestamp=1732014906489Num uniques 859; 2024-11-19T11:15:06,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:15:06,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efdfe7abed14b673b10bbcdae68405c7 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-19T11:15:06,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014905852, max timestamp=1732014906489Num uniques 859; 2024-11-19T11:15:06,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:15:06,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014905852, max timestamp=1732014906489Num uniques 859; 2024-11-19T11:15:06,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:15:06,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=A 2024-11-19T11:15:06,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:06,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=B 2024-11-19T11:15:06,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:06,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=C 2024-11-19T11:15:06,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:06,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119973e51e95377458fa56ad520a8fecd69_efdfe7abed14b673b10bbcdae68405c7 is 50, key is test_row_0/A:col10/1732014906490/Put/seqid=0 2024-11-19T11:15:06,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742041_1217 (size=17534) 2024-11-19T11:15:06,511 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:06,515 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119973e51e95377458fa56ad520a8fecd69_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119973e51e95377458fa56ad520a8fecd69_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:06,516 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/5ab533fe50704542acad3a230290adc1, store: [table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:15:06,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/5ab533fe50704542acad3a230290adc1 is 175, key is test_row_0/A:col10/1732014906490/Put/seqid=0 2024-11-19T11:15:06,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742042_1218 (size=48639) 2024-11-19T11:15:06,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732014906516, max timestamp=1732014906529Num uniques 859; 2024-11-19T11:15:06,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T11:15:06,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732014906516, max timestamp=1732014906529Num uniques 859; 2024-11-19T11:15:06,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732014906516, max timestamp=1732014906529Num uniques 859; 2024-11-19T11:15:06,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T11:15:06,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T11:15:06,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:06,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014966560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:06,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:06,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014966562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014966563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:06,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:06,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014966667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:06,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:06,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014966677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:06,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014966677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:06,744 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/7ba73e0526444f51a5b27132dfcff42f as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/7ba73e0526444f51a5b27132dfcff42f 2024-11-19T11:15:06,748 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efdfe7abed14b673b10bbcdae68405c7/B of efdfe7abed14b673b10bbcdae68405c7 into 7ba73e0526444f51a5b27132dfcff42f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:06,748 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:15:06,748 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., storeName=efdfe7abed14b673b10bbcdae68405c7/B, priority=13, startTime=1732014906310; duration=0sec 2024-11-19T11:15:06,748 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:06,748 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efdfe7abed14b673b10bbcdae68405c7:B 2024-11-19T11:15:06,748 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:06,749 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:06,749 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): efdfe7abed14b673b10bbcdae68405c7/C is initiating minor compaction (all files) 2024-11-19T11:15:06,749 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efdfe7abed14b673b10bbcdae68405c7/C in TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:06,749 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/42f5791e139c42528cf3d7477a966c7d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/fd885cb6345f467cbf3e721ef668b588, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/039a1c8f6d404e0aa756e7046567d593] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp, totalSize=36.6 K 2024-11-19T11:15:06,749 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 42f5791e139c42528cf3d7477a966c7d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732014901369 2024-11-19T11:15:06,750 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting fd885cb6345f467cbf3e721ef668b588, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732014902566 2024-11-19T11:15:06,750 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 039a1c8f6d404e0aa756e7046567d593, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732014904692 2024-11-19T11:15:06,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:06,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41250 deadline: 1732014966747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:06,753 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_1' in efdfe7abed14b673b10bbcdae68405c7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:15:06,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:06,755 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efdfe7abed14b673b10bbcdae68405c7#A#compaction#179 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:06,756 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/28b8c2350e77488bb75607188ef15fd6 is 175, key is test_row_0/A:col10/1732014904692/Put/seqid=0 2024-11-19T11:15:06,757 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efdfe7abed14b673b10bbcdae68405c7#C#compaction#182 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:06,758 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/18e758547d04448383ba662aa7af922f is 50, key is test_row_0/C:col10/1732014904692/Put/seqid=0 2024-11-19T11:15:06,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742043_1219 (size=31937) 2024-11-19T11:15:06,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742044_1220 (size=12983) 2024-11-19T11:15:06,770 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/28b8c2350e77488bb75607188ef15fd6 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/28b8c2350e77488bb75607188ef15fd6 2024-11-19T11:15:06,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:06,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41260 deadline: 1732014966767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:06,772 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_2' in efdfe7abed14b673b10bbcdae68405c7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:15:06,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:06,777 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efdfe7abed14b673b10bbcdae68405c7/A of efdfe7abed14b673b10bbcdae68405c7 into 28b8c2350e77488bb75607188ef15fd6(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:06,777 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:15:06,777 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., storeName=efdfe7abed14b673b10bbcdae68405c7/A, priority=13, startTime=1732014906310; duration=0sec 2024-11-19T11:15:06,777 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:06,777 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efdfe7abed14b673b10bbcdae68405c7:A 2024-11-19T11:15:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T11:15:06,848 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:15:06,848 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:15:06,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-19T11:15:06,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T11:15:06,850 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:15:06,850 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:15:06,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:15:06,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:06,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014966878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:06,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:06,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014966887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:06,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:06,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:06,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014966887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:06,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:06,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:06,926 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=371, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/5ab533fe50704542acad3a230290adc1 2024-11-19T11:15:06,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/329cc3d49b134aee88d2c2704606fd65 is 50, key is test_row_0/B:col10/1732014906490/Put/seqid=0 2024-11-19T11:15:06,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742045_1221 (size=12301) 2024-11-19T11:15:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T11:15:07,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T11:15:07,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:07,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:15:07,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:07,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:07,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:07,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T11:15:07,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:07,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:15:07,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:07,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:07,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T11:15:07,170 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/18e758547d04448383ba662aa7af922f as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/18e758547d04448383ba662aa7af922f 2024-11-19T11:15:07,175 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efdfe7abed14b673b10bbcdae68405c7/C of efdfe7abed14b673b10bbcdae68405c7 into 18e758547d04448383ba662aa7af922f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:07,175 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:15:07,175 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., storeName=efdfe7abed14b673b10bbcdae68405c7/C, priority=13, startTime=1732014906310; duration=0sec 2024-11-19T11:15:07,175 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:07,176 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efdfe7abed14b673b10bbcdae68405c7:C 2024-11-19T11:15:07,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014967187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:07,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:07,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:07,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:07,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014967198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:07,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014967198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:07,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T11:15:07,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:07,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:15:07,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:07,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:07,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:07,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/329cc3d49b134aee88d2c2704606fd65 2024-11-19T11:15:07,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/bddce714db0a4d8e936306b65f847f15 is 50, key is test_row_0/C:col10/1732014906490/Put/seqid=0 2024-11-19T11:15:07,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742046_1222 (size=12301) 2024-11-19T11:15:07,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T11:15:07,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:07,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:15:07,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:07,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:07,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T11:15:07,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T11:15:07,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:07,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:15:07,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:07,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:07,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:07,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:07,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014967697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:07,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:07,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:07,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:07,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:07,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014967718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:07,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:07,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:07,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014967718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:07,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:07,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:07,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:07,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:07,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:07,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/bddce714db0a4d8e936306b65f847f15 2024-11-19T11:15:07,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/5ab533fe50704542acad3a230290adc1 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/5ab533fe50704542acad3a230290adc1 2024-11-19T11:15:07,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/5ab533fe50704542acad3a230290adc1, entries=250, sequenceid=371, filesize=47.5 K 2024-11-19T11:15:07,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/329cc3d49b134aee88d2c2704606fd65 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/329cc3d49b134aee88d2c2704606fd65 2024-11-19T11:15:07,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/329cc3d49b134aee88d2c2704606fd65, entries=150, sequenceid=371, filesize=12.0 K 2024-11-19T11:15:07,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/bddce714db0a4d8e936306b65f847f15 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/bddce714db0a4d8e936306b65f847f15 2024-11-19T11:15:07,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T11:15:07,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:07,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:15:07,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:07,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:07,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:07,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:07,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/bddce714db0a4d8e936306b65f847f15, entries=150, sequenceid=371, filesize=12.0 K 2024-11-19T11:15:07,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.03 KB/108576, currentSize=147.60 KB/151140 for efdfe7abed14b673b10bbcdae68405c7 in 1279ms, sequenceid=371, compaction requested=false 2024-11-19T11:15:07,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:15:07,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T11:15:07,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:07,920 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing efdfe7abed14b673b10bbcdae68405c7 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-19T11:15:07,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=A 2024-11-19T11:15:07,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:07,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=B 2024-11-19T11:15:07,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:07,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=C 2024-11-19T11:15:07,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:07,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f97ef5f3e9454c26b36118f6061b728b_efdfe7abed14b673b10bbcdae68405c7 is 50, key is test_row_0/A:col10/1732014906562/Put/seqid=0 2024-11-19T11:15:07,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742047_1223 (size=12454) 2024-11-19T11:15:07,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T11:15:08,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:08,335 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f97ef5f3e9454c26b36118f6061b728b_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f97ef5f3e9454c26b36118f6061b728b_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:08,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/88107a3829e445dbbb35f10e4385531a, store: [table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:15:08,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/88107a3829e445dbbb35f10e4385531a is 175, key is test_row_0/A:col10/1732014906562/Put/seqid=0 2024-11-19T11:15:08,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742048_1224 (size=31255) 2024-11-19T11:15:08,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. as already flushing 2024-11-19T11:15:08,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:08,740 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=410, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/88107a3829e445dbbb35f10e4385531a 2024-11-19T11:15:08,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/9614ea80052645b8b1b8933549ab094f is 50, key is test_row_0/B:col10/1732014906562/Put/seqid=0 2024-11-19T11:15:08,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742049_1225 (size=12301) 2024-11-19T11:15:08,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:08,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014968762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:08,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:08,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014968762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:08,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:08,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014968762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:08,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:08,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014968877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014968877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:08,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014968878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:08,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:08,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:08,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:08,933 INFO [Thread-675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-675. 2024-11-19T11:15:08,933 DEBUG [Thread-675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:15:08,933 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:08,933 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:08,933 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:15:08,937 INFO [Thread-677 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-677. 2024-11-19T11:15:08,937 DEBUG [Thread-677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:15:08,937 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:08,938 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:08,938 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:15:08,945 INFO [Thread-681 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-681. 2024-11-19T11:15:08,945 DEBUG [Thread-681 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:15:08,945 DEBUG [Thread-681 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:08,945 DEBUG [Thread-681 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:08,946 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:15:08,946 INFO [Thread-679 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-679. 2024-11-19T11:15:08,946 DEBUG [Thread-679 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:15:08,946 DEBUG [Thread-679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:08,946 DEBUG [Thread-679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:08,946 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:15:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T11:15:09,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:09,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014969088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014969088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:09,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014969088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:09,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:09,151 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/9614ea80052645b8b1b8933549ab094f 2024-11-19T11:15:09,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/ed40cd1542704b7da49f64e50bc84d9e is 50, key is test_row_0/C:col10/1732014906562/Put/seqid=0 2024-11-19T11:15:09,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742050_1226 (size=12301) 2024-11-19T11:15:09,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:09,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:09,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41258 deadline: 1732014969397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41270 deadline: 1732014969397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41248 deadline: 1732014969397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:09,562 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/ed40cd1542704b7da49f64e50bc84d9e 2024-11-19T11:15:09,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/88107a3829e445dbbb35f10e4385531a as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/88107a3829e445dbbb35f10e4385531a 2024-11-19T11:15:09,599 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/88107a3829e445dbbb35f10e4385531a, entries=150, sequenceid=410, filesize=30.5 K 2024-11-19T11:15:09,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/9614ea80052645b8b1b8933549ab094f as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/9614ea80052645b8b1b8933549ab094f 2024-11-19T11:15:09,603 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/9614ea80052645b8b1b8933549ab094f, entries=150, sequenceid=410, filesize=12.0 K 2024-11-19T11:15:09,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/ed40cd1542704b7da49f64e50bc84d9e as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/ed40cd1542704b7da49f64e50bc84d9e 2024-11-19T11:15:09,607 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/ed40cd1542704b7da49f64e50bc84d9e, entries=150, sequenceid=410, filesize=12.0 K 2024-11-19T11:15:09,608 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for efdfe7abed14b673b10bbcdae68405c7 in 1688ms, sequenceid=410, compaction requested=true 2024-11-19T11:15:09,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:15:09,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:09,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-19T11:15:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-19T11:15:09,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-19T11:15:09,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7580 sec 2024-11-19T11:15:09,612 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.7630 sec 2024-11-19T11:15:09,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:09,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efdfe7abed14b673b10bbcdae68405c7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T11:15:09,914 INFO [Thread-668 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-668. 2024-11-19T11:15:09,914 DEBUG [Thread-668 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:15:09,914 DEBUG [Thread-668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:09,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=A 2024-11-19T11:15:09,915 INFO [Thread-672 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-672. 2024-11-19T11:15:09,915 DEBUG [Thread-668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:09,915 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:15:09,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:09,915 INFO [Thread-670 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-670. 2024-11-19T11:15:09,915 DEBUG [Thread-672 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:15:09,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=B 2024-11-19T11:15:09,915 DEBUG [Thread-670 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:15:09,915 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:09,915 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:09,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:09,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=C 2024-11-19T11:15:09,915 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:09,915 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:09,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:09,915 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:15:09,915 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:15:09,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119185bf8fac0854cfe9b88d377c9a98ebc_efdfe7abed14b673b10bbcdae68405c7 is 50, key is test_row_0/A:col10/1732014908736/Put/seqid=0 2024-11-19T11:15:09,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742051_1227 (size=12454) 2024-11-19T11:15:10,330 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:10,340 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119185bf8fac0854cfe9b88d377c9a98ebc_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119185bf8fac0854cfe9b88d377c9a98ebc_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:10,343 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/a71d9cf15a314aeaa6e621d24d67a444, store: [table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:15:10,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/a71d9cf15a314aeaa6e621d24d67a444 is 175, key is test_row_0/A:col10/1732014908736/Put/seqid=0 2024-11-19T11:15:10,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742052_1228 (size=31255) 2024-11-19T11:15:10,749 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=422, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/a71d9cf15a314aeaa6e621d24d67a444 2024-11-19T11:15:10,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/ba3423c5356246e18d6591e9c7977b47 is 50, key is test_row_0/B:col10/1732014908736/Put/seqid=0 2024-11-19T11:15:10,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742053_1229 (size=12301) 2024-11-19T11:15:10,788 INFO [Thread-666 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-666. 2024-11-19T11:15:10,788 DEBUG [Thread-666 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:15:10,788 DEBUG [Thread-666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:10,788 INFO [Thread-664 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-664. 2024-11-19T11:15:10,788 DEBUG [Thread-664 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:15:10,789 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:10,789 DEBUG [Thread-666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:10,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:15:10,789 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:10,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:15:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T11:15:11,008 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:15:11,008 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T11:15:11,008 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-19T11:15:11,008 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-19T11:15:11,008 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-19T11:15:11,008 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-19T11:15:11,008 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-19T11:15:11,008 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T11:15:11,008 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2861 2024-11-19T11:15:11,008 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2921 2024-11-19T11:15:11,008 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T11:15:11,008 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1201 2024-11-19T11:15:11,008 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3599 rows 2024-11-19T11:15:11,008 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1209 2024-11-19T11:15:11,008 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3623 rows 2024-11-19T11:15:11,009 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T11:15:11,009 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T11:15:11,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:11,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:11,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:15:11,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T11:15:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T11:15:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T11:15:11,016 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014911016"}]},"ts":"1732014911016"} 2024-11-19T11:15:11,019 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T11:15:11,019 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T11:15:11,020 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T11:15:11,022 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efdfe7abed14b673b10bbcdae68405c7, UNASSIGN}] 2024-11-19T11:15:11,023 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efdfe7abed14b673b10bbcdae68405c7, UNASSIGN 2024-11-19T11:15:11,024 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=efdfe7abed14b673b10bbcdae68405c7, regionState=CLOSING, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:15:11,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=efdfe7abed14b673b10bbcdae68405c7, UNASSIGN because future has completed 2024-11-19T11:15:11,026 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T11:15:11,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640}] 2024-11-19T11:15:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T11:15:11,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/ba3423c5356246e18d6591e9c7977b47 2024-11-19T11:15:11,179 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:11,180 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T11:15:11,180 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing efdfe7abed14b673b10bbcdae68405c7, disabling compactions & flushes 2024-11-19T11:15:11,180 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:11,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/e520e2cbdff94d6fb8a16608e02023d9 is 50, key is test_row_0/C:col10/1732014908736/Put/seqid=0 2024-11-19T11:15:11,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742054_1230 (size=12301) 2024-11-19T11:15:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T11:15:11,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/e520e2cbdff94d6fb8a16608e02023d9 2024-11-19T11:15:11,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/a71d9cf15a314aeaa6e621d24d67a444 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/a71d9cf15a314aeaa6e621d24d67a444 2024-11-19T11:15:11,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/a71d9cf15a314aeaa6e621d24d67a444, entries=150, sequenceid=422, filesize=30.5 K 2024-11-19T11:15:11,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/ba3423c5356246e18d6591e9c7977b47 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/ba3423c5356246e18d6591e9c7977b47 2024-11-19T11:15:11,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/ba3423c5356246e18d6591e9c7977b47, entries=150, sequenceid=422, filesize=12.0 K 2024-11-19T11:15:11,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/e520e2cbdff94d6fb8a16608e02023d9 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/e520e2cbdff94d6fb8a16608e02023d9 2024-11-19T11:15:11,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/e520e2cbdff94d6fb8a16608e02023d9, entries=150, sequenceid=422, filesize=12.0 K 2024-11-19T11:15:11,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=13.42 KB/13740 for efdfe7abed14b673b10bbcdae68405c7 in 1705ms, sequenceid=422, compaction requested=true 2024-11-19T11:15:11,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efdfe7abed14b673b10bbcdae68405c7: 2024-11-19T11:15:11,618 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:11,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efdfe7abed14b673b10bbcdae68405c7:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:15:11,618 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:11,618 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. because compaction request was cancelled 2024-11-19T11:15:11,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:11,618 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efdfe7abed14b673b10bbcdae68405c7:A 2024-11-19T11:15:11,618 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. after waiting 0 ms 2024-11-19T11:15:11,618 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. because compaction request was cancelled 2024-11-19T11:15:11,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efdfe7abed14b673b10bbcdae68405c7:B, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:15:11,618 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:11,618 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efdfe7abed14b673b10bbcdae68405c7:B 2024-11-19T11:15:11,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:11,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efdfe7abed14b673b10bbcdae68405c7:C, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:15:11,618 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. because compaction request was cancelled 2024-11-19T11:15:11,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:11,618 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing efdfe7abed14b673b10bbcdae68405c7 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-19T11:15:11,618 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efdfe7abed14b673b10bbcdae68405c7:C 2024-11-19T11:15:11,619 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=A 2024-11-19T11:15:11,619 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:11,619 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=B 2024-11-19T11:15:11,619 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:11,619 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efdfe7abed14b673b10bbcdae68405c7, store=C 2024-11-19T11:15:11,619 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:11,624 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195fb92ab11d6f4d21928117b3cecdd59e_efdfe7abed14b673b10bbcdae68405c7 is 50, key is test_row_1/A:col10/1732014910787/Put/seqid=0 2024-11-19T11:15:11,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742055_1231 (size=9914) 2024-11-19T11:15:11,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T11:15:12,030 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:12,037 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195fb92ab11d6f4d21928117b3cecdd59e_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195fb92ab11d6f4d21928117b3cecdd59e_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:12,038 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/e0c41104656f4649bc2b5973181c5d97, store: [table=TestAcidGuarantees family=A region=efdfe7abed14b673b10bbcdae68405c7] 2024-11-19T11:15:12,038 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/e0c41104656f4649bc2b5973181c5d97 is 175, key is test_row_1/A:col10/1732014910787/Put/seqid=0 2024-11-19T11:15:12,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742056_1232 (size=22561) 2024-11-19T11:15:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T11:15:12,444 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=427, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/e0c41104656f4649bc2b5973181c5d97 2024-11-19T11:15:12,458 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/313f0dcdc7ae4c8791a7cdf20f09f1a7 is 50, key is test_row_1/B:col10/1732014910787/Put/seqid=0 2024-11-19T11:15:12,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742057_1233 (size=9857) 2024-11-19T11:15:12,863 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/313f0dcdc7ae4c8791a7cdf20f09f1a7 2024-11-19T11:15:12,880 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/80d309834b6d4da5bb74539d74e1fc7d is 50, key is test_row_1/C:col10/1732014910787/Put/seqid=0 2024-11-19T11:15:12,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742058_1234 (size=9857) 2024-11-19T11:15:13,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T11:15:13,286 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/80d309834b6d4da5bb74539d74e1fc7d 2024-11-19T11:15:13,296 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/A/e0c41104656f4649bc2b5973181c5d97 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/e0c41104656f4649bc2b5973181c5d97 2024-11-19T11:15:13,300 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/e0c41104656f4649bc2b5973181c5d97, entries=100, sequenceid=427, filesize=22.0 K 2024-11-19T11:15:13,300 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/B/313f0dcdc7ae4c8791a7cdf20f09f1a7 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/313f0dcdc7ae4c8791a7cdf20f09f1a7 2024-11-19T11:15:13,304 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/313f0dcdc7ae4c8791a7cdf20f09f1a7, entries=100, sequenceid=427, filesize=9.6 K 2024-11-19T11:15:13,305 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/.tmp/C/80d309834b6d4da5bb74539d74e1fc7d as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/80d309834b6d4da5bb74539d74e1fc7d 2024-11-19T11:15:13,308 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/80d309834b6d4da5bb74539d74e1fc7d, entries=100, sequenceid=427, filesize=9.6 K 2024-11-19T11:15:13,309 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for efdfe7abed14b673b10bbcdae68405c7 in 1691ms, sequenceid=427, compaction requested=true 2024-11-19T11:15:13,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/20e00b82b0744d92a1048fade156fd44, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/7b35c853e3b04c8693c864d1525fd9d0, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/cf36185268204a2da92fbf5452fd4679, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/edd5b08884134d74ab41fa38124a2e82, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/4f383e08229d451fa9279e32b6a14399, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/3cc84bd51f9e43bcab23936b75456c79, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/06e205d592cf4609aa4dbf1d9bdef969, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/7521958460c44954b4d36e316352e710, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/19462fd78f384776bb9bb95e87126cb4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/a2119629aae747128c043f4bc6c7cf46, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/bfa594b878d1406d86f00bac0d26f5da, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/8759eaca84584e978994f779ba543104, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/a57b7ea1480d487381f4e741cc6313c9, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/9e6614e80b9741f4af5090e3f9eaf29b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/e1a643230fa6436eb38c044eb140b5fd, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/99f7f6d2497c45c4aca5cf4bc8ad3283, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/473c9613f4854ef0a9d5afcae0cb39bf, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/1a6702ff54e7434cb6bf1b0255ffcb73, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/443d23d1396a4c31afe257f8867381f4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/e1b292d9077c4ec18edbfc71b3f01987] to archive 2024-11-19T11:15:13,310 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T11:15:13,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/20e00b82b0744d92a1048fade156fd44 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/20e00b82b0744d92a1048fade156fd44 2024-11-19T11:15:13,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/7b35c853e3b04c8693c864d1525fd9d0 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/7b35c853e3b04c8693c864d1525fd9d0 2024-11-19T11:15:13,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/cf36185268204a2da92fbf5452fd4679 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/cf36185268204a2da92fbf5452fd4679 2024-11-19T11:15:13,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/edd5b08884134d74ab41fa38124a2e82 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/edd5b08884134d74ab41fa38124a2e82 2024-11-19T11:15:13,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/4f383e08229d451fa9279e32b6a14399 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/4f383e08229d451fa9279e32b6a14399 2024-11-19T11:15:13,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/3cc84bd51f9e43bcab23936b75456c79 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/3cc84bd51f9e43bcab23936b75456c79 2024-11-19T11:15:13,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/06e205d592cf4609aa4dbf1d9bdef969 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/06e205d592cf4609aa4dbf1d9bdef969 2024-11-19T11:15:13,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/7521958460c44954b4d36e316352e710 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/7521958460c44954b4d36e316352e710 2024-11-19T11:15:13,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/19462fd78f384776bb9bb95e87126cb4 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/19462fd78f384776bb9bb95e87126cb4 2024-11-19T11:15:13,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/a2119629aae747128c043f4bc6c7cf46 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/a2119629aae747128c043f4bc6c7cf46 2024-11-19T11:15:13,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/bfa594b878d1406d86f00bac0d26f5da to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/bfa594b878d1406d86f00bac0d26f5da 2024-11-19T11:15:13,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/8759eaca84584e978994f779ba543104 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/8759eaca84584e978994f779ba543104 2024-11-19T11:15:13,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/a57b7ea1480d487381f4e741cc6313c9 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/a57b7ea1480d487381f4e741cc6313c9 2024-11-19T11:15:13,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/9e6614e80b9741f4af5090e3f9eaf29b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/9e6614e80b9741f4af5090e3f9eaf29b 2024-11-19T11:15:13,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/e1a643230fa6436eb38c044eb140b5fd to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/e1a643230fa6436eb38c044eb140b5fd 2024-11-19T11:15:13,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/99f7f6d2497c45c4aca5cf4bc8ad3283 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/99f7f6d2497c45c4aca5cf4bc8ad3283 2024-11-19T11:15:13,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/473c9613f4854ef0a9d5afcae0cb39bf to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/473c9613f4854ef0a9d5afcae0cb39bf 2024-11-19T11:15:13,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/1a6702ff54e7434cb6bf1b0255ffcb73 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/1a6702ff54e7434cb6bf1b0255ffcb73 2024-11-19T11:15:13,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/443d23d1396a4c31afe257f8867381f4 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/443d23d1396a4c31afe257f8867381f4 2024-11-19T11:15:13,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/e1b292d9077c4ec18edbfc71b3f01987 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/e1b292d9077c4ec18edbfc71b3f01987 2024-11-19T11:15:13,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/42e0df460dd942529ec0fb1e7a5cbcc8, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/6625c3da5905429bb72b53304ddda0be, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/72a389ec6d27424381cd18f24294bac7, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/4baf5a218787414f94399456f2521efd, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/7cb6f01298304a3cafd71b00480225b3, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/a465b825bb86409eb40907212ad45d4d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/3551ffe615a94c35a88328300766ec92, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/db64e9ab44604ba9b001e42530f1d2a7, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/3bcb6fde91f040d198146b6e462f1ead, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/c7cb91d518e6487cb1f1896ae10e868b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/3037e75bae4f4bc790430ed144a5f89c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/d63077e6febb4ed7b6bc6d19fd431afc, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/205066e086e54eb4856213eb2fa65561, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/f609e3c13ca3414795244b26dea0b418, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/d8c22d2ae3714604b554df98bb918954, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/e92b27da74414dbd8fab16af85c27009, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/4fc6164f0ada44629978932b6eaa400d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/6723203fffdf4d328ebbd22bb25f5d87, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/823d24272a854321aeac12a25292f731, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/f400a7c0c5b54f0eb4b467f900501063] to archive 2024-11-19T11:15:13,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T11:15:13,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/42e0df460dd942529ec0fb1e7a5cbcc8 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/42e0df460dd942529ec0fb1e7a5cbcc8 2024-11-19T11:15:13,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/6625c3da5905429bb72b53304ddda0be to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/6625c3da5905429bb72b53304ddda0be 2024-11-19T11:15:13,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/72a389ec6d27424381cd18f24294bac7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/72a389ec6d27424381cd18f24294bac7 2024-11-19T11:15:13,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/4baf5a218787414f94399456f2521efd to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/4baf5a218787414f94399456f2521efd 2024-11-19T11:15:13,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/7cb6f01298304a3cafd71b00480225b3 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/7cb6f01298304a3cafd71b00480225b3 2024-11-19T11:15:13,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/a465b825bb86409eb40907212ad45d4d to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/a465b825bb86409eb40907212ad45d4d 2024-11-19T11:15:13,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/3551ffe615a94c35a88328300766ec92 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/3551ffe615a94c35a88328300766ec92 2024-11-19T11:15:13,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/db64e9ab44604ba9b001e42530f1d2a7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/db64e9ab44604ba9b001e42530f1d2a7 2024-11-19T11:15:13,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/3bcb6fde91f040d198146b6e462f1ead to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/3bcb6fde91f040d198146b6e462f1ead 2024-11-19T11:15:13,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/c7cb91d518e6487cb1f1896ae10e868b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/c7cb91d518e6487cb1f1896ae10e868b 2024-11-19T11:15:13,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/3037e75bae4f4bc790430ed144a5f89c to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/3037e75bae4f4bc790430ed144a5f89c 2024-11-19T11:15:13,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/d63077e6febb4ed7b6bc6d19fd431afc to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/d63077e6febb4ed7b6bc6d19fd431afc 2024-11-19T11:15:13,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/205066e086e54eb4856213eb2fa65561 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/205066e086e54eb4856213eb2fa65561 2024-11-19T11:15:13,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/f609e3c13ca3414795244b26dea0b418 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/f609e3c13ca3414795244b26dea0b418 2024-11-19T11:15:13,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/d8c22d2ae3714604b554df98bb918954 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/d8c22d2ae3714604b554df98bb918954 2024-11-19T11:15:13,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/e92b27da74414dbd8fab16af85c27009 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/e92b27da74414dbd8fab16af85c27009 2024-11-19T11:15:13,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/4fc6164f0ada44629978932b6eaa400d to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/4fc6164f0ada44629978932b6eaa400d 2024-11-19T11:15:13,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/6723203fffdf4d328ebbd22bb25f5d87 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/6723203fffdf4d328ebbd22bb25f5d87 2024-11-19T11:15:13,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/823d24272a854321aeac12a25292f731 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/823d24272a854321aeac12a25292f731 2024-11-19T11:15:13,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/f400a7c0c5b54f0eb4b467f900501063 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/f400a7c0c5b54f0eb4b467f900501063 2024-11-19T11:15:13,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/a550cf7e1066440ca5b93cc7906ee82e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/2f02ceca12b646b3be1242227b6e28cf, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/72b01cabc4cd4c919c4cc99c7b4f324d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/36e01e666a0c4a2d81cb905b9f52c8a6, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/13ec8655a70d435fa614053e45163d83, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/03514653ef44444ba64e8c218641b791, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/1ebd602cbbba4cf8b363eddd35ddada1, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/51188b3feba94b468caf448d5ee54682, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/4f47ee358bb341df835fa15669b90314, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/2faaa5f5b003431d873b226a402817eb, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/67d5d4196a744f5a828342e7f633f7cf, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/b27fb25e37274738bd0f6ed42f4b144b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/af0764ae21fc41248843e96093e6fb13, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/4fa1a763b6e64567afa93fdf6d8243eb, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/757e26c880b54ae6bcae475a7e9eab8d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/e3d97da94b8c4c4caeb4e850d948eb2d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/42f5791e139c42528cf3d7477a966c7d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/53e6c63706194fcaabd315fc228c3291, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/fd885cb6345f467cbf3e721ef668b588, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/039a1c8f6d404e0aa756e7046567d593] to archive 2024-11-19T11:15:13,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T11:15:13,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/a550cf7e1066440ca5b93cc7906ee82e to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/a550cf7e1066440ca5b93cc7906ee82e 2024-11-19T11:15:13,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/2f02ceca12b646b3be1242227b6e28cf to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/2f02ceca12b646b3be1242227b6e28cf 2024-11-19T11:15:13,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/72b01cabc4cd4c919c4cc99c7b4f324d to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/72b01cabc4cd4c919c4cc99c7b4f324d 2024-11-19T11:15:13,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/36e01e666a0c4a2d81cb905b9f52c8a6 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/36e01e666a0c4a2d81cb905b9f52c8a6 2024-11-19T11:15:13,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/13ec8655a70d435fa614053e45163d83 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/13ec8655a70d435fa614053e45163d83 2024-11-19T11:15:13,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/03514653ef44444ba64e8c218641b791 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/03514653ef44444ba64e8c218641b791 2024-11-19T11:15:13,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/1ebd602cbbba4cf8b363eddd35ddada1 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/1ebd602cbbba4cf8b363eddd35ddada1 2024-11-19T11:15:13,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/51188b3feba94b468caf448d5ee54682 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/51188b3feba94b468caf448d5ee54682 2024-11-19T11:15:13,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/4f47ee358bb341df835fa15669b90314 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/4f47ee358bb341df835fa15669b90314 2024-11-19T11:15:13,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/2faaa5f5b003431d873b226a402817eb to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/2faaa5f5b003431d873b226a402817eb 2024-11-19T11:15:13,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/67d5d4196a744f5a828342e7f633f7cf to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/67d5d4196a744f5a828342e7f633f7cf 2024-11-19T11:15:13,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/b27fb25e37274738bd0f6ed42f4b144b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/b27fb25e37274738bd0f6ed42f4b144b 2024-11-19T11:15:13,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/af0764ae21fc41248843e96093e6fb13 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/af0764ae21fc41248843e96093e6fb13 2024-11-19T11:15:13,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/4fa1a763b6e64567afa93fdf6d8243eb to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/4fa1a763b6e64567afa93fdf6d8243eb 2024-11-19T11:15:13,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/757e26c880b54ae6bcae475a7e9eab8d to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/757e26c880b54ae6bcae475a7e9eab8d 2024-11-19T11:15:13,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/e3d97da94b8c4c4caeb4e850d948eb2d to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/e3d97da94b8c4c4caeb4e850d948eb2d 2024-11-19T11:15:13,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/42f5791e139c42528cf3d7477a966c7d to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/42f5791e139c42528cf3d7477a966c7d 2024-11-19T11:15:13,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/53e6c63706194fcaabd315fc228c3291 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/53e6c63706194fcaabd315fc228c3291 2024-11-19T11:15:13,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/fd885cb6345f467cbf3e721ef668b588 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/fd885cb6345f467cbf3e721ef668b588 2024-11-19T11:15:13,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/039a1c8f6d404e0aa756e7046567d593 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/039a1c8f6d404e0aa756e7046567d593 2024-11-19T11:15:13,378 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/recovered.edits/430.seqid, newMaxSeqId=430, maxSeqId=4 2024-11-19T11:15:13,378 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. 2024-11-19T11:15:13,378 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for efdfe7abed14b673b10bbcdae68405c7: Waiting for close lock at 1732014911180Running coprocessor pre-close hooks at 1732014911180Disabling compacts and flushes for region at 1732014911180Disabling writes for close at 1732014911618 (+438 ms)Obtaining lock to block concurrent updates at 1732014911618Preparing flush snapshotting stores in efdfe7abed14b673b10bbcdae68405c7 at 1732014911618Finished memstore snapshotting TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1732014911619 (+1 ms)Flushing stores of TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7. at 1732014911619Flushing efdfe7abed14b673b10bbcdae68405c7/A: creating writer at 1732014911619Flushing mob file efdfe7abed14b673b10bbcdae68405c7/A: appending metadata at 1732014911623 (+4 ms)Flushing mob file efdfe7abed14b673b10bbcdae68405c7/A: closing flushed file at 1732014911623Flushing efdfe7abed14b673b10bbcdae68405c7/A: appending metadata at 1732014912038 (+415 ms)Flushing efdfe7abed14b673b10bbcdae68405c7/A: closing flushed file at 1732014912038Flushing efdfe7abed14b673b10bbcdae68405c7/B: creating writer at 1732014912454 (+416 ms)Flushing efdfe7abed14b673b10bbcdae68405c7/B: appending metadata at 1732014912457 (+3 ms)Flushing efdfe7abed14b673b10bbcdae68405c7/B: closing flushed file at 1732014912458 (+1 ms)Flushing efdfe7abed14b673b10bbcdae68405c7/C: creating writer at 1732014912874 (+416 ms)Flushing efdfe7abed14b673b10bbcdae68405c7/C: appending metadata at 1732014912879 (+5 ms)Flushing efdfe7abed14b673b10bbcdae68405c7/C: closing flushed file at 1732014912879Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ebb87e8: reopening flushed file at 1732014913295 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c0c9b6d: reopening flushed file at 1732014913300 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57ac98ad: reopening flushed file at 1732014913304 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for efdfe7abed14b673b10bbcdae68405c7 in 1691ms, sequenceid=427, compaction requested=true at 1732014913309 (+5 ms)Writing region close event to WAL at 1732014913375 (+66 ms)Running coprocessor post-close hooks at 1732014913378 (+3 ms)Closed at 1732014913378 2024-11-19T11:15:13,380 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:13,381 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=efdfe7abed14b673b10bbcdae68405c7, regionState=CLOSED 2024-11-19T11:15:13,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 because future has completed 2024-11-19T11:15:13,384 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-19T11:15:13,384 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure efdfe7abed14b673b10bbcdae68405c7, server=16c88966c0af,35833,1732014855640 in 2.3560 sec 2024-11-19T11:15:13,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-19T11:15:13,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efdfe7abed14b673b10bbcdae68405c7, UNASSIGN in 2.3620 sec 2024-11-19T11:15:13,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-19T11:15:13,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.3660 sec 2024-11-19T11:15:13,389 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014913389"}]},"ts":"1732014913389"} 2024-11-19T11:15:13,390 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T11:15:13,390 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T11:15:13,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.3800 sec 2024-11-19T11:15:13,566 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T11:15:14,759 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T11:15:15,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T11:15:15,189 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:15:15,190 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T11:15:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:15:15,195 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:15:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T11:15:15,198 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:15:15,201 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:15,204 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A, FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B, FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C, FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/recovered.edits] 2024-11-19T11:15:15,208 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/28b8c2350e77488bb75607188ef15fd6 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/28b8c2350e77488bb75607188ef15fd6 2024-11-19T11:15:15,210 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/5ab533fe50704542acad3a230290adc1 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/5ab533fe50704542acad3a230290adc1 2024-11-19T11:15:15,212 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/88107a3829e445dbbb35f10e4385531a to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/88107a3829e445dbbb35f10e4385531a 2024-11-19T11:15:15,214 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/a71d9cf15a314aeaa6e621d24d67a444 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/a71d9cf15a314aeaa6e621d24d67a444 2024-11-19T11:15:15,217 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/e0c41104656f4649bc2b5973181c5d97 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/A/e0c41104656f4649bc2b5973181c5d97 2024-11-19T11:15:15,221 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/313f0dcdc7ae4c8791a7cdf20f09f1a7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/313f0dcdc7ae4c8791a7cdf20f09f1a7 2024-11-19T11:15:15,223 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/329cc3d49b134aee88d2c2704606fd65 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/329cc3d49b134aee88d2c2704606fd65 2024-11-19T11:15:15,225 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/7ba73e0526444f51a5b27132dfcff42f to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/7ba73e0526444f51a5b27132dfcff42f 2024-11-19T11:15:15,226 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/9614ea80052645b8b1b8933549ab094f to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/9614ea80052645b8b1b8933549ab094f 2024-11-19T11:15:15,227 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/ba3423c5356246e18d6591e9c7977b47 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/B/ba3423c5356246e18d6591e9c7977b47 2024-11-19T11:15:15,229 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/18e758547d04448383ba662aa7af922f to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/18e758547d04448383ba662aa7af922f 2024-11-19T11:15:15,230 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/80d309834b6d4da5bb74539d74e1fc7d to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/80d309834b6d4da5bb74539d74e1fc7d 2024-11-19T11:15:15,231 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/bddce714db0a4d8e936306b65f847f15 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/bddce714db0a4d8e936306b65f847f15 2024-11-19T11:15:15,232 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/e520e2cbdff94d6fb8a16608e02023d9 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/e520e2cbdff94d6fb8a16608e02023d9 2024-11-19T11:15:15,233 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/ed40cd1542704b7da49f64e50bc84d9e to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/C/ed40cd1542704b7da49f64e50bc84d9e 2024-11-19T11:15:15,234 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/recovered.edits/430.seqid to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7/recovered.edits/430.seqid 2024-11-19T11:15:15,235 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:15,235 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T11:15:15,235 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T11:15:15,236 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T11:15:15,239 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119185bf8fac0854cfe9b88d377c9a98ebc_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119185bf8fac0854cfe9b88d377c9a98ebc_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:15,240 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191c8dec15d29d43b0bb9a20efc05e0a20_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191c8dec15d29d43b0bb9a20efc05e0a20_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:15,241 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192d59c435d2944f13bc21c14047df8001_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192d59c435d2944f13bc21c14047df8001_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:15,242 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193053e3e8ad0d4c21b9723ab15206c154_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193053e3e8ad0d4c21b9723ab15206c154_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:15,243 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195000da6bb11d43ce9546cef4a2439f9b_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195000da6bb11d43ce9546cef4a2439f9b_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:15,243 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195143bdcb49d3464fbf3562f994439ff1_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195143bdcb49d3464fbf3562f994439ff1_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:15,244 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195fb92ab11d6f4d21928117b3cecdd59e_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195fb92ab11d6f4d21928117b3cecdd59e_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:15,245 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119605333a8d3b24ab7961c1b531220e625_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119605333a8d3b24ab7961c1b531220e625_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:15,246 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196f66de225d4b4c4d902a9464ffe47183_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196f66de225d4b4c4d902a9464ffe47183_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:15,246 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111975261449ad89408893a0f58ae4c79ef5_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111975261449ad89408893a0f58ae4c79ef5_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:15,247 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198b12024780bc4ab9937c0a73d012cdec_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198b12024780bc4ab9937c0a73d012cdec_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:15,248 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199147cfa1ca314702a5d1859db263e2b1_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199147cfa1ca314702a5d1859db263e2b1_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:15,249 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119973e51e95377458fa56ad520a8fecd69_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119973e51e95377458fa56ad520a8fecd69_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:15,250 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199e43b80408e9420891c7b1daadb84ccb_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199e43b80408e9420891c7b1daadb84ccb_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:15,251 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bc662b9ac272468cbc36e0b16797186e_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bc662b9ac272468cbc36e0b16797186e_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:15,252 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bee4abe4285e4ddeb936264f0d761604_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bee4abe4285e4ddeb936264f0d761604_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:15,253 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c04800c0dc1c4552b29bf18f878e1a3e_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c04800c0dc1c4552b29bf18f878e1a3e_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:15,254 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119eed80fef64f24a628e49e7629f0bc0ac_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119eed80fef64f24a628e49e7629f0bc0ac_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:15,254 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f97ef5f3e9454c26b36118f6061b728b_efdfe7abed14b673b10bbcdae68405c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f97ef5f3e9454c26b36118f6061b728b_efdfe7abed14b673b10bbcdae68405c7 2024-11-19T11:15:15,255 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T11:15:15,257 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:15:15,260 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T11:15:15,262 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T11:15:15,263 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:15:15,263 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T11:15:15,264 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732014915263"}]},"ts":"9223372036854775807"} 2024-11-19T11:15:15,266 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T11:15:15,266 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => efdfe7abed14b673b10bbcdae68405c7, NAME => 'TestAcidGuarantees,,1732014886746.efdfe7abed14b673b10bbcdae68405c7.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T11:15:15,266 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T11:15:15,266 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732014915266"}]},"ts":"9223372036854775807"} 2024-11-19T11:15:15,268 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T11:15:15,269 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:15:15,270 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 79 msec 2024-11-19T11:15:15,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T11:15:15,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T11:15:15,311 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:15:15,319 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 248) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1969214525_22 at /127.0.0.1:44212 [Waiting for operation #1131] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-855594354_22 at /127.0.0.1:34152 [Waiting for operation #1563] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1969214525_22 at /127.0.0.1:44200 [Waiting for operation #1139] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/cluster_5de41c1e-6874-2d94-bb17-966f646923cf/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/cluster_5de41c1e-6874-2d94-bb17-966f646923cf/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-855594354_22 at /127.0.0.1:44148 [Waiting for operation #1174] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=380 (was 464), ProcessCount=11 (was 11), AvailableMemoryMB=11768 (was 11845) 2024-11-19T11:15:15,330 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=380, ProcessCount=11, AvailableMemoryMB=11768 2024-11-19T11:15:15,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T11:15:15,331 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:15:15,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T11:15:15,334 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T11:15:15,334 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:15,334 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-11-19T11:15:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-19T11:15:15,335 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T11:15:15,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742059_1235 (size=1061) 2024-11-19T11:15:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-19T11:15:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-19T11:15:15,748 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094 2024-11-19T11:15:15,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742060_1236 (size=53) 2024-11-19T11:15:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-19T11:15:16,158 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:15:16,158 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ae1beb1ad4dbb1059f68b61e2fd94bf8, disabling compactions & flushes 2024-11-19T11:15:16,158 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:16,158 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:16,159 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. after waiting 1 ms 2024-11-19T11:15:16,159 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:16,159 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:16,159 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: Waiting for close lock at 1732014916158Disabling compacts and flushes for region at 1732014916158Disabling writes for close at 1732014916159 (+1 ms)Writing region close event to WAL at 1732014916159Closed at 1732014916159 2024-11-19T11:15:16,162 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T11:15:16,163 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732014916162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732014916162"}]},"ts":"1732014916162"} 2024-11-19T11:15:16,166 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T11:15:16,167 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T11:15:16,167 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014916167"}]},"ts":"1732014916167"} 2024-11-19T11:15:16,168 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T11:15:16,169 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae1beb1ad4dbb1059f68b61e2fd94bf8, ASSIGN}] 2024-11-19T11:15:16,170 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae1beb1ad4dbb1059f68b61e2fd94bf8, ASSIGN 2024-11-19T11:15:16,171 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae1beb1ad4dbb1059f68b61e2fd94bf8, ASSIGN; state=OFFLINE, location=16c88966c0af,35833,1732014855640; forceNewPlan=false, retain=false 2024-11-19T11:15:16,321 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=ae1beb1ad4dbb1059f68b61e2fd94bf8, regionState=OPENING, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:15:16,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae1beb1ad4dbb1059f68b61e2fd94bf8, ASSIGN because future has completed 2024-11-19T11:15:16,324 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640}] 2024-11-19T11:15:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-19T11:15:16,479 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:16,479 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} 2024-11-19T11:15:16,479 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:16,480 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:15:16,480 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:16,480 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:16,481 INFO [StoreOpener-ae1beb1ad4dbb1059f68b61e2fd94bf8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:16,482 INFO [StoreOpener-ae1beb1ad4dbb1059f68b61e2fd94bf8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:15:16,482 INFO [StoreOpener-ae1beb1ad4dbb1059f68b61e2fd94bf8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae1beb1ad4dbb1059f68b61e2fd94bf8 columnFamilyName A 2024-11-19T11:15:16,482 DEBUG [StoreOpener-ae1beb1ad4dbb1059f68b61e2fd94bf8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:16,482 INFO [StoreOpener-ae1beb1ad4dbb1059f68b61e2fd94bf8-1 {}] regionserver.HStore(327): Store=ae1beb1ad4dbb1059f68b61e2fd94bf8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:15:16,482 INFO [StoreOpener-ae1beb1ad4dbb1059f68b61e2fd94bf8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:16,483 INFO [StoreOpener-ae1beb1ad4dbb1059f68b61e2fd94bf8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:15:16,483 INFO [StoreOpener-ae1beb1ad4dbb1059f68b61e2fd94bf8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae1beb1ad4dbb1059f68b61e2fd94bf8 columnFamilyName B 2024-11-19T11:15:16,484 DEBUG [StoreOpener-ae1beb1ad4dbb1059f68b61e2fd94bf8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:16,484 INFO [StoreOpener-ae1beb1ad4dbb1059f68b61e2fd94bf8-1 {}] regionserver.HStore(327): Store=ae1beb1ad4dbb1059f68b61e2fd94bf8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:15:16,484 INFO [StoreOpener-ae1beb1ad4dbb1059f68b61e2fd94bf8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:16,485 INFO [StoreOpener-ae1beb1ad4dbb1059f68b61e2fd94bf8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:15:16,485 INFO [StoreOpener-ae1beb1ad4dbb1059f68b61e2fd94bf8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae1beb1ad4dbb1059f68b61e2fd94bf8 columnFamilyName C 2024-11-19T11:15:16,485 DEBUG [StoreOpener-ae1beb1ad4dbb1059f68b61e2fd94bf8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:16,485 INFO [StoreOpener-ae1beb1ad4dbb1059f68b61e2fd94bf8-1 {}] regionserver.HStore(327): Store=ae1beb1ad4dbb1059f68b61e2fd94bf8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:15:16,486 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:16,486 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:16,486 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:16,487 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:16,488 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:16,488 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:16,488 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T11:15:16,489 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:16,491 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T11:15:16,491 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened ae1beb1ad4dbb1059f68b61e2fd94bf8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68796077, jitterRate=0.025141432881355286}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T11:15:16,491 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:16,492 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: Running coprocessor pre-open hook at 1732014916480Writing region info on filesystem at 1732014916480Initializing all the Stores at 1732014916480Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014916480Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014916481 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014916481Cleaning up temporary data from old regions at 1732014916488 (+7 ms)Running coprocessor post-open hooks at 1732014916491 (+3 ms)Region opened successfully at 1732014916492 (+1 ms) 2024-11-19T11:15:16,493 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., pid=59, masterSystemTime=1732014916476 2024-11-19T11:15:16,494 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:16,494 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:16,494 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=ae1beb1ad4dbb1059f68b61e2fd94bf8, regionState=OPEN, openSeqNum=2, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:15:16,496 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 because future has completed 2024-11-19T11:15:16,498 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-19T11:15:16,498 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 in 173 msec 2024-11-19T11:15:16,500 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-19T11:15:16,500 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae1beb1ad4dbb1059f68b61e2fd94bf8, ASSIGN in 329 msec 2024-11-19T11:15:16,501 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T11:15:16,501 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014916501"}]},"ts":"1732014916501"} 2024-11-19T11:15:16,502 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T11:15:16,503 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T11:15:16,504 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1720 sec 2024-11-19T11:15:17,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-19T11:15:17,477 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:15:17,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,478 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:15:17,478 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:15:17,479 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:15:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:15:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:15:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e0388da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:15:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:15:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:17,480 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:15:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50f94bc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,481 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:15:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:15:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:15:17,483 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:15:17,484 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T11:15:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 16c88966c0af,41441,1732014854424 2024-11-19T11:15:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3e0f0acc 2024-11-19T11:15:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T11:15:17,486 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T11:15:17,487 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,487 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:15:17,487 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:15:17,488 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:15:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:15:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:15:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@339959da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:15:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:15:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:17,489 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:15:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ab7966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,491 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:15:17,491 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:15:17,492 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:15:17,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:15:17,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:15:17,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ce7111b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:15:17,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:15:17,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:17,493 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:15:17,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17f99ef0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,494 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:15:17,494 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:15:17,495 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:15:17,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:15:17,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:15:17,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28651e0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:15:17,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:15:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:17,496 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:15:17,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23bb8f3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,498 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:15:17,498 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:15:17,498 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:15:17,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:15:17,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:15:17,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@169ff17d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:15:17,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:15:17,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:17,500 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:15:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@741bf0a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,503 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:15:17,504 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:15:17,504 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:15:17,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:15:17,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:15:17,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5960694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:15:17,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:15:17,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:17,506 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:15:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34625c19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,507 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,507 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:15:17,507 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:15:17,507 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:15:17,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:15:17,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:15:17,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d2b5211, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:15:17,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:15:17,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:17,509 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:15:17,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@507d41eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,510 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:15:17,510 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:15:17,510 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:15:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:15:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:15:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21e65ac1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:15:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:15:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:17,512 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:15:17,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e933378, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,513 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:15:17,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:15:17,513 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:15:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:15:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:15:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260bcf6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:15:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:15:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:17,515 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:15:17,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63455081, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,516 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:15:17,516 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:15:17,517 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:15:17,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:15:17,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:15:17,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23de9b35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:15:17,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:15:17,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:17,518 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:15:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ce69b4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,519 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,519 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:15:17,519 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:15:17,520 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:15:17,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:15:17,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:15:17,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@459b9868, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:15:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:15:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:17,521 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:15:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a96c2be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:17,522 DEBUG [Thread-1121 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:15:17,522 DEBUG [Thread-1106 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:15:17,522 DEBUG [Thread-1110 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:15:17,522 DEBUG [Thread-1115 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:15:17,523 DEBUG [Thread-1119 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:15:17,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:15:17,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:15:17,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:15:17,523 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:15:17,523 DEBUG [Thread-1102 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:15:17,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:15:17,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:15:17,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:15:17,524 DEBUG [Thread-1104 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:15:17,524 DEBUG [Thread-1117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:15:17,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:15:17,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-19T11:15:17,524 DEBUG [Thread-1108 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:15:17,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:15:17,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:15:17,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:15:17,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:15:17,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:15:17,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:15:17,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:15:17,525 DEBUG [Thread-1117 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:15:17,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:15:17,525 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:15:17,525 DEBUG [Thread-1117 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:15:17,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:15:17,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:15:17,526 DEBUG [Thread-1108 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:15:17,526 DEBUG [Thread-1108 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:15:17,526 DEBUG [Thread-1113 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:15:17,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T11:15:17,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:15:17,527 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:15:17,527 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:15:17,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:15:17,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:15:17,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:15:17,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:15:17,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:15:17,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:15:17,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:15:17,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:15:17,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:15:17,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:15:17,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:15:17,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:15:17,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:15:17,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:15:17,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:15:17,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:15:17,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:15:17,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:15:17,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T11:15:17,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:17,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:17,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:17,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:17,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:17,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:17,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:17,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/67764469e6ef4e0ab7b3cc4cf5f3646c is 50, key is test_row_0/A:col10/1732014917540/Put/seqid=0 2024-11-19T11:15:17,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:17,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014977568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:17,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:17,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:17,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014977569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:17,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:17,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014977569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:17,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:17,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:17,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:17,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014977570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:17,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:17,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014977571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:17,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:17,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:17,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742061_1237 (size=12001) 2024-11-19T11:15:17,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/67764469e6ef4e0ab7b3cc4cf5f3646c 2024-11-19T11:15:17,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/93bfcde5b1674b3fa3424be3d09db528 is 50, key is test_row_0/B:col10/1732014917540/Put/seqid=0 2024-11-19T11:15:17,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742062_1238 (size=12001) 2024-11-19T11:15:17,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/93bfcde5b1674b3fa3424be3d09db528 2024-11-19T11:15:17,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T11:15:17,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/e387d356496c4f4d9f1383fc7d15b97d is 50, key is test_row_0/C:col10/1732014917540/Put/seqid=0 2024-11-19T11:15:17,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742063_1239 (size=12001) 2024-11-19T11:15:17,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:17,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014977678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:17,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:17,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014977678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:17,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:17,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:17,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:17,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014977678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:17,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:17,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014977678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:17,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-19T11:15:17,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:17,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:17,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:17,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:17,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014977678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:17,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:17,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:17,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:17,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:17,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:17,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:17,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-19T11:15:17,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:17,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:17,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:17,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:17,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T11:15:17,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:17,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014977888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014977888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:17,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014977888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014977888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:17,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014977888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:17,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-19T11:15:17,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:17,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:17,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:17,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:17,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:18,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/e387d356496c4f4d9f1383fc7d15b97d 2024-11-19T11:15:18,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/67764469e6ef4e0ab7b3cc4cf5f3646c as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/67764469e6ef4e0ab7b3cc4cf5f3646c 2024-11-19T11:15:18,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/67764469e6ef4e0ab7b3cc4cf5f3646c, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T11:15:18,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/93bfcde5b1674b3fa3424be3d09db528 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/93bfcde5b1674b3fa3424be3d09db528 2024-11-19T11:15:18,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/93bfcde5b1674b3fa3424be3d09db528, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T11:15:18,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/e387d356496c4f4d9f1383fc7d15b97d as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/e387d356496c4f4d9f1383fc7d15b97d 2024-11-19T11:15:18,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/e387d356496c4f4d9f1383fc7d15b97d, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T11:15:18,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 553ms, sequenceid=13, compaction requested=false 2024-11-19T11:15:18,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:18,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-19T11:15:18,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:18,147 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T11:15:18,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:18,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:18,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:18,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:18,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:18,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:18,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/821f195e780043aca27adbb771452130 is 50, key is test_row_0/A:col10/1732014917568/Put/seqid=0 2024-11-19T11:15:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T11:15:18,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742064_1240 (size=9657) 2024-11-19T11:15:18,160 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/821f195e780043aca27adbb771452130 2024-11-19T11:15:18,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/ca7209e01dd248cfb272cf26dacef384 is 50, key is test_row_0/B:col10/1732014917568/Put/seqid=0 2024-11-19T11:15:18,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742065_1241 (size=9657) 2024-11-19T11:15:18,171 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/ca7209e01dd248cfb272cf26dacef384 2024-11-19T11:15:18,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/c4c177daa1cd4c15b1a64c1a2222d6fd is 50, key is test_row_0/C:col10/1732014917568/Put/seqid=0 2024-11-19T11:15:18,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742066_1242 (size=9657) 2024-11-19T11:15:18,184 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/c4c177daa1cd4c15b1a64c1a2222d6fd 2024-11-19T11:15:18,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/821f195e780043aca27adbb771452130 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/821f195e780043aca27adbb771452130 2024-11-19T11:15:18,196 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/821f195e780043aca27adbb771452130, entries=100, sequenceid=33, filesize=9.4 K 2024-11-19T11:15:18,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/ca7209e01dd248cfb272cf26dacef384 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/ca7209e01dd248cfb272cf26dacef384 2024-11-19T11:15:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:18,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:18,202 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/ca7209e01dd248cfb272cf26dacef384, entries=100, sequenceid=33, filesize=9.4 K 2024-11-19T11:15:18,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/c4c177daa1cd4c15b1a64c1a2222d6fd as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/c4c177daa1cd4c15b1a64c1a2222d6fd 2024-11-19T11:15:18,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014978204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:18,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014978204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:18,209 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/c4c177daa1cd4c15b1a64c1a2222d6fd, entries=100, sequenceid=33, filesize=9.4 K 2024-11-19T11:15:18,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014978205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,210 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 63ms, sequenceid=33, compaction requested=false 2024-11-19T11:15:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:18,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:18,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:18,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-19T11:15:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-19T11:15:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:18,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T11:15:18,213 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-19T11:15:18,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:18,213 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 684 msec 2024-11-19T11:15:18,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:18,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:18,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:18,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:18,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:18,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 690 msec 2024-11-19T11:15:18,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/275099fd77f44219b0c80d2f7856de3a is 50, key is test_row_0/A:col10/1732014918206/Put/seqid=0 2024-11-19T11:15:18,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742067_1243 (size=12001) 2024-11-19T11:15:18,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014978259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:18,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:18,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014978260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:18,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:18,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014978317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:18,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014978317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014978317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:18,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014978368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:18,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:18,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014978368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:18,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014978527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:18,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:18,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014978527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:18,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:18,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:18,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014978527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:18,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:18,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014978577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:18,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:18,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014978577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:18,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:18,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:18,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/275099fd77f44219b0c80d2f7856de3a 2024-11-19T11:15:18,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/a9f906f41c8e4d3fb3ecfa0c08c6b81a is 50, key is test_row_0/B:col10/1732014918206/Put/seqid=0 2024-11-19T11:15:18,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742068_1244 (size=12001) 2024-11-19T11:15:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T11:15:18,667 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:15:18,667 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:15:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-19T11:15:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T11:15:18,670 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:15:18,671 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:15:18,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:15:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T11:15:18,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T11:15:18,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:18,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:18,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:18,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:18,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:18,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014978837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:18,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014978838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014978838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:18,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014978887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:18,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:18,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014978888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:18,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:18,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:18,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:18,905 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T11:15:18,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T11:15:18,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:18,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:18,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:18,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:18,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T11:15:19,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/a9f906f41c8e4d3fb3ecfa0c08c6b81a 2024-11-19T11:15:19,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/14704031a463467ab7c8f6879af211eb is 50, key is test_row_0/C:col10/1732014918206/Put/seqid=0 2024-11-19T11:15:19,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742069_1245 (size=12001) 2024-11-19T11:15:19,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T11:15:19,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:19,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:19,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:19,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:19,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:19,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T11:15:19,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:19,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:19,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:19,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:19,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:19,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:19,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T11:15:19,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014979347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:19,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014979347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:19,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014979347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:19,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:19,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014979398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:19,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:19,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:19,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:19,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014979398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:19,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:19,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:19,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:19,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:19,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T11:15:19,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:19,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:19,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:19,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:19,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:19,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/14704031a463467ab7c8f6879af211eb 2024-11-19T11:15:19,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/275099fd77f44219b0c80d2f7856de3a as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/275099fd77f44219b0c80d2f7856de3a 2024-11-19T11:15:19,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/275099fd77f44219b0c80d2f7856de3a, entries=150, sequenceid=46, filesize=11.7 K 2024-11-19T11:15:19,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/a9f906f41c8e4d3fb3ecfa0c08c6b81a as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/a9f906f41c8e4d3fb3ecfa0c08c6b81a 2024-11-19T11:15:19,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/a9f906f41c8e4d3fb3ecfa0c08c6b81a, entries=150, sequenceid=46, filesize=11.7 K 2024-11-19T11:15:19,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/14704031a463467ab7c8f6879af211eb as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/14704031a463467ab7c8f6879af211eb 2024-11-19T11:15:19,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/14704031a463467ab7c8f6879af211eb, entries=150, sequenceid=46, filesize=11.7 K 2024-11-19T11:15:19,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 1287ms, sequenceid=46, compaction requested=true 2024-11-19T11:15:19,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:19,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:15:19,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:19,499 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:19,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:15:19,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:19,500 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:19,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:15:19,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:19,503 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:19,503 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:19,503 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/B is initiating minor compaction (all files) 2024-11-19T11:15:19,503 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/A is initiating minor compaction (all files) 2024-11-19T11:15:19,503 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/A in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:19,503 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/67764469e6ef4e0ab7b3cc4cf5f3646c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/821f195e780043aca27adbb771452130, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/275099fd77f44219b0c80d2f7856de3a] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=32.9 K 2024-11-19T11:15:19,504 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/B in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:19,504 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/93bfcde5b1674b3fa3424be3d09db528, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/ca7209e01dd248cfb272cf26dacef384, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/a9f906f41c8e4d3fb3ecfa0c08c6b81a] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=32.9 K 2024-11-19T11:15:19,504 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 93bfcde5b1674b3fa3424be3d09db528, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732014917532 2024-11-19T11:15:19,504 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67764469e6ef4e0ab7b3cc4cf5f3646c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732014917532 2024-11-19T11:15:19,505 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting ca7209e01dd248cfb272cf26dacef384, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732014917568 2024-11-19T11:15:19,505 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 821f195e780043aca27adbb771452130, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732014917568 2024-11-19T11:15:19,505 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 275099fd77f44219b0c80d2f7856de3a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732014918206 2024-11-19T11:15:19,505 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting a9f906f41c8e4d3fb3ecfa0c08c6b81a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732014918206 2024-11-19T11:15:19,526 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#A#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:19,526 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/bd80dc53232d4f1da7efa6b43cb3ec89 is 50, key is test_row_0/A:col10/1732014918206/Put/seqid=0 2024-11-19T11:15:19,537 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#B#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:19,538 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/fcbdd7ae788c4920ad8556e0d1cee4d4 is 50, key is test_row_0/B:col10/1732014918206/Put/seqid=0 2024-11-19T11:15:19,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742070_1246 (size=12104) 2024-11-19T11:15:19,570 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/bd80dc53232d4f1da7efa6b43cb3ec89 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/bd80dc53232d4f1da7efa6b43cb3ec89 2024-11-19T11:15:19,578 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/A of ae1beb1ad4dbb1059f68b61e2fd94bf8 into bd80dc53232d4f1da7efa6b43cb3ec89(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:19,578 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:19,578 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/A, priority=13, startTime=1732014919499; duration=0sec 2024-11-19T11:15:19,578 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:19,578 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:A 2024-11-19T11:15:19,578 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:19,581 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:19,582 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/C is initiating minor compaction (all files) 2024-11-19T11:15:19,582 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/C in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:19,582 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/e387d356496c4f4d9f1383fc7d15b97d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/c4c177daa1cd4c15b1a64c1a2222d6fd, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/14704031a463467ab7c8f6879af211eb] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=32.9 K 2024-11-19T11:15:19,582 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting e387d356496c4f4d9f1383fc7d15b97d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732014917532 2024-11-19T11:15:19,583 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4c177daa1cd4c15b1a64c1a2222d6fd, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732014917568 2024-11-19T11:15:19,583 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14704031a463467ab7c8f6879af211eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732014918206 2024-11-19T11:15:19,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T11:15:19,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:19,591 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T11:15:19,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:19,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:19,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:19,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:19,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:19,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:19,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742071_1247 (size=12104) 2024-11-19T11:15:19,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/c1559d5ef47a4eb3a5847f04e5d47335 is 50, key is test_row_0/A:col10/1732014918259/Put/seqid=0 2024-11-19T11:15:19,615 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#C#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:19,615 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/bbf7d7b4d29e46088952e1a0bbffe7c3 is 50, key is test_row_0/C:col10/1732014918206/Put/seqid=0 2024-11-19T11:15:19,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742072_1248 (size=12001) 2024-11-19T11:15:19,659 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/c1559d5ef47a4eb3a5847f04e5d47335 2024-11-19T11:15:19,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742073_1249 (size=12104) 2024-11-19T11:15:19,674 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/bbf7d7b4d29e46088952e1a0bbffe7c3 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/bbf7d7b4d29e46088952e1a0bbffe7c3 2024-11-19T11:15:19,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/a4e3a7102e82424a90346eec0da165cc is 50, key is test_row_0/B:col10/1732014918259/Put/seqid=0 2024-11-19T11:15:19,683 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/C of ae1beb1ad4dbb1059f68b61e2fd94bf8 into bbf7d7b4d29e46088952e1a0bbffe7c3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:19,683 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:19,683 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/C, priority=13, startTime=1732014919500; duration=0sec 2024-11-19T11:15:19,683 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:19,683 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:C 2024-11-19T11:15:19,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742074_1250 (size=12001) 2024-11-19T11:15:19,689 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/a4e3a7102e82424a90346eec0da165cc 2024-11-19T11:15:19,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/a079e8b87fac46c687f279caf1994ddc is 50, key is test_row_0/C:col10/1732014918259/Put/seqid=0 2024-11-19T11:15:19,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742075_1251 (size=12001) 2024-11-19T11:15:19,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T11:15:20,009 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/fcbdd7ae788c4920ad8556e0d1cee4d4 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/fcbdd7ae788c4920ad8556e0d1cee4d4 2024-11-19T11:15:20,014 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/B of ae1beb1ad4dbb1059f68b61e2fd94bf8 into fcbdd7ae788c4920ad8556e0d1cee4d4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:20,014 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:20,014 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/B, priority=13, startTime=1732014919499; duration=0sec 2024-11-19T11:15:20,014 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:20,014 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:B 2024-11-19T11:15:20,152 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/a079e8b87fac46c687f279caf1994ddc 2024-11-19T11:15:20,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/c1559d5ef47a4eb3a5847f04e5d47335 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/c1559d5ef47a4eb3a5847f04e5d47335 2024-11-19T11:15:20,164 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/c1559d5ef47a4eb3a5847f04e5d47335, entries=150, sequenceid=67, filesize=11.7 K 2024-11-19T11:15:20,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/a4e3a7102e82424a90346eec0da165cc as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/a4e3a7102e82424a90346eec0da165cc 2024-11-19T11:15:20,170 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/a4e3a7102e82424a90346eec0da165cc, entries=150, sequenceid=67, filesize=11.7 K 2024-11-19T11:15:20,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/a079e8b87fac46c687f279caf1994ddc as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/a079e8b87fac46c687f279caf1994ddc 2024-11-19T11:15:20,180 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/a079e8b87fac46c687f279caf1994ddc, entries=150, sequenceid=67, filesize=11.7 K 2024-11-19T11:15:20,181 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 590ms, sequenceid=67, compaction requested=false 2024-11-19T11:15:20,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:20,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:20,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-19T11:15:20,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-19T11:15:20,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-19T11:15:20,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5110 sec 2024-11-19T11:15:20,190 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.5180 sec 2024-11-19T11:15:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:20,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T11:15:20,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:20,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:20,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:20,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:20,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:20,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:20,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/9db7b8fa016e4c37aeb03b87e0e4d5a9 is 50, key is test_row_0/A:col10/1732014920367/Put/seqid=0 2024-11-19T11:15:20,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742076_1252 (size=14341) 2024-11-19T11:15:20,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/9db7b8fa016e4c37aeb03b87e0e4d5a9 2024-11-19T11:15:20,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/2dad97dd207f495b99cba9204a4c66d0 is 50, key is test_row_0/B:col10/1732014920367/Put/seqid=0 2024-11-19T11:15:20,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732014920394, max timestamp=1732014920402Num uniques 859; 2024-11-19T11:15:20,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T11:15:20,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732014920394, max timestamp=1732014920402Num uniques 859; 2024-11-19T11:15:20,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T11:15:20,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732014920394, max timestamp=1732014920402Num uniques 859; 2024-11-19T11:15:20,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T11:15:20,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742077_1253 (size=12001) 2024-11-19T11:15:20,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/2dad97dd207f495b99cba9204a4c66d0 2024-11-19T11:15:20,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/b6ec4078f5194a1fb222cdad1ba6f415 is 50, key is test_row_0/C:col10/1732014920367/Put/seqid=0 2024-11-19T11:15:20,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014980431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:20,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014980432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:20,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742078_1254 (size=12001) 2024-11-19T11:15:20,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014980433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:20,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014980435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:20,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014980457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:20,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014980568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:20,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014980568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014980568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014980568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:20,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014980568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:20,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:20,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014980778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:20,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:20,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:20,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014980778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:20,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:20,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014980778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014980778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014980778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T11:15:20,818 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:15:20,819 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:15:20,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-19T11:15:20,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T11:15:20,821 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:15:20,823 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:15:20,824 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:15:20,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/b6ec4078f5194a1fb222cdad1ba6f415 2024-11-19T11:15:20,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/9db7b8fa016e4c37aeb03b87e0e4d5a9 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/9db7b8fa016e4c37aeb03b87e0e4d5a9 2024-11-19T11:15:20,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/9db7b8fa016e4c37aeb03b87e0e4d5a9, entries=200, sequenceid=80, filesize=14.0 K 2024-11-19T11:15:20,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/2dad97dd207f495b99cba9204a4c66d0 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/2dad97dd207f495b99cba9204a4c66d0 2024-11-19T11:15:20,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/2dad97dd207f495b99cba9204a4c66d0, entries=150, sequenceid=80, filesize=11.7 K 2024-11-19T11:15:20,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/b6ec4078f5194a1fb222cdad1ba6f415 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/b6ec4078f5194a1fb222cdad1ba6f415 2024-11-19T11:15:20,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/b6ec4078f5194a1fb222cdad1ba6f415, entries=150, sequenceid=80, filesize=11.7 K 2024-11-19T11:15:20,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=124.84 KB/127835 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 507ms, sequenceid=80, compaction requested=true 2024-11-19T11:15:20,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:20,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:15:20,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:20,874 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:20,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:15:20,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:20,875 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:20,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:15:20,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:20,875 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:20,875 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:20,875 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/A is initiating minor compaction (all files) 2024-11-19T11:15:20,875 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/B is initiating minor compaction (all files) 2024-11-19T11:15:20,875 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/B in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:20,875 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/A in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:20,876 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/fcbdd7ae788c4920ad8556e0d1cee4d4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/a4e3a7102e82424a90346eec0da165cc, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/2dad97dd207f495b99cba9204a4c66d0] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=35.3 K 2024-11-19T11:15:20,876 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/bd80dc53232d4f1da7efa6b43cb3ec89, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/c1559d5ef47a4eb3a5847f04e5d47335, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/9db7b8fa016e4c37aeb03b87e0e4d5a9] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=37.5 K 2024-11-19T11:15:20,876 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting fcbdd7ae788c4920ad8556e0d1cee4d4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732014918206 2024-11-19T11:15:20,876 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd80dc53232d4f1da7efa6b43cb3ec89, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732014918206 2024-11-19T11:15:20,876 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting a4e3a7102e82424a90346eec0da165cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732014918252 2024-11-19T11:15:20,876 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1559d5ef47a4eb3a5847f04e5d47335, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732014918252 2024-11-19T11:15:20,876 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dad97dd207f495b99cba9204a4c66d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732014920357 2024-11-19T11:15:20,876 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9db7b8fa016e4c37aeb03b87e0e4d5a9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732014920357 2024-11-19T11:15:20,883 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#A#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:20,884 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#B#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:20,884 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/1f736e7129d54e258cbddf5a55533299 is 50, key is test_row_0/A:col10/1732014920367/Put/seqid=0 2024-11-19T11:15:20,884 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/f625d0d012894534b558f8c0387f3c24 is 50, key is test_row_0/B:col10/1732014920367/Put/seqid=0 2024-11-19T11:15:20,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742079_1255 (size=12207) 2024-11-19T11:15:20,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742080_1256 (size=12207) 2024-11-19T11:15:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T11:15:20,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T11:15:20,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:20,977 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=124.84 KB heapSize=354.38 KB 2024-11-19T11:15:20,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:20,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:20,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:20,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:20,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:20,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:20,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/325ee4ecceef40bf93f338d9d6a887ff is 50, key is test_row_0/A:col10/1732014920418/Put/seqid=0 2024-11-19T11:15:20,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742081_1257 (size=12001) 2024-11-19T11:15:21,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:21,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:21,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:21,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014981097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014981097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:21,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014981099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:21,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:21,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014981106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:21,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:21,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014981106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:21,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:21,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T11:15:21,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:21,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014981217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:21,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014981217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:21,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014981218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:21,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014981218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014981218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:21,300 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/1f736e7129d54e258cbddf5a55533299 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/1f736e7129d54e258cbddf5a55533299 2024-11-19T11:15:21,307 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/A of ae1beb1ad4dbb1059f68b61e2fd94bf8 into 1f736e7129d54e258cbddf5a55533299(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:21,307 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:21,307 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/A, priority=13, startTime=1732014920874; duration=0sec 2024-11-19T11:15:21,307 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:21,307 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:A 2024-11-19T11:15:21,307 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:21,308 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:21,309 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/C is initiating minor compaction (all files) 2024-11-19T11:15:21,309 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/C in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:21,309 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/bbf7d7b4d29e46088952e1a0bbffe7c3, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/a079e8b87fac46c687f279caf1994ddc, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/b6ec4078f5194a1fb222cdad1ba6f415] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=35.3 K 2024-11-19T11:15:21,309 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbf7d7b4d29e46088952e1a0bbffe7c3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732014918206 2024-11-19T11:15:21,311 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting a079e8b87fac46c687f279caf1994ddc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732014918252 2024-11-19T11:15:21,312 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6ec4078f5194a1fb222cdad1ba6f415, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732014920357 2024-11-19T11:15:21,313 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/f625d0d012894534b558f8c0387f3c24 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/f625d0d012894534b558f8c0387f3c24 2024-11-19T11:15:21,323 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/B of ae1beb1ad4dbb1059f68b61e2fd94bf8 into f625d0d012894534b558f8c0387f3c24(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:21,323 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:21,323 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/B, priority=13, startTime=1732014920874; duration=0sec 2024-11-19T11:15:21,323 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:21,323 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:B 2024-11-19T11:15:21,327 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#C#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:21,327 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/0d3c64898aff40f2ba4fa1eefbdce613 is 50, key is test_row_0/C:col10/1732014920367/Put/seqid=0 2024-11-19T11:15:21,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742082_1258 (size=12207) 2024-11-19T11:15:21,356 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/0d3c64898aff40f2ba4fa1eefbdce613 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/0d3c64898aff40f2ba4fa1eefbdce613 2024-11-19T11:15:21,375 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/C of ae1beb1ad4dbb1059f68b61e2fd94bf8 into 0d3c64898aff40f2ba4fa1eefbdce613(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:21,376 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:21,376 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/C, priority=13, startTime=1732014920875; duration=0sec 2024-11-19T11:15:21,376 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:21,376 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:C 2024-11-19T11:15:21,393 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/325ee4ecceef40bf93f338d9d6a887ff 2024-11-19T11:15:21,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/128e2a3e0f294d8fa9d43606c38dbdab is 50, key is test_row_0/B:col10/1732014920418/Put/seqid=0 2024-11-19T11:15:21,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014981427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:21,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014981427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014981428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014981428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:21,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:21,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:21,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014981438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:21,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:21,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742083_1259 (size=12001) 2024-11-19T11:15:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T11:15:21,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014981738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:21,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:21,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014981738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:21,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:21,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:21,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:21,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014981738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:21,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:21,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:21,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014981738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:21,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:21,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:21,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014981747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:21,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:21,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:21,843 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/128e2a3e0f294d8fa9d43606c38dbdab 2024-11-19T11:15:21,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/ffb6c0abdde9409bb25e2f1064ea31a5 is 50, key is test_row_0/C:col10/1732014920418/Put/seqid=0 2024-11-19T11:15:21,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742084_1260 (size=12001) 2024-11-19T11:15:21,882 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/ffb6c0abdde9409bb25e2f1064ea31a5 2024-11-19T11:15:21,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/325ee4ecceef40bf93f338d9d6a887ff as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/325ee4ecceef40bf93f338d9d6a887ff 2024-11-19T11:15:21,894 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/325ee4ecceef40bf93f338d9d6a887ff, entries=150, sequenceid=115, filesize=11.7 K 2024-11-19T11:15:21,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/128e2a3e0f294d8fa9d43606c38dbdab as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/128e2a3e0f294d8fa9d43606c38dbdab 2024-11-19T11:15:21,902 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/128e2a3e0f294d8fa9d43606c38dbdab, entries=150, sequenceid=115, filesize=11.7 K 2024-11-19T11:15:21,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/ffb6c0abdde9409bb25e2f1064ea31a5 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/ffb6c0abdde9409bb25e2f1064ea31a5 2024-11-19T11:15:21,909 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/ffb6c0abdde9409bb25e2f1064ea31a5, entries=150, sequenceid=115, filesize=11.7 K 2024-11-19T11:15:21,910 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~124.84 KB/127835, heapSize ~354.34 KB/362840, currentSize=53.67 KB/54960 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 933ms, sequenceid=115, compaction requested=false 2024-11-19T11:15:21,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:21,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:21,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-19T11:15:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-19T11:15:21,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-19T11:15:21,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0890 sec 2024-11-19T11:15:21,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.0990 sec 2024-11-19T11:15:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T11:15:21,958 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:15:21,958 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:15:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-19T11:15:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T11:15:21,963 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:15:21,963 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:15:21,964 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:15:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T11:15:22,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T11:15:22,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:22,116 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T11:15:22,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:22,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:22,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:22,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:22,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:22,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:22,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/f447dfb6161b477ab0a1934de6004a3e is 50, key is test_row_0/A:col10/1732014921102/Put/seqid=0 2024-11-19T11:15:22,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742085_1261 (size=12001) 2024-11-19T11:15:22,156 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/f447dfb6161b477ab0a1934de6004a3e 2024-11-19T11:15:22,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/cd224fc0e4dc4ac5997890f73cef932f is 50, key is test_row_0/B:col10/1732014921102/Put/seqid=0 2024-11-19T11:15:22,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742086_1262 (size=12001) 2024-11-19T11:15:22,207 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/cd224fc0e4dc4ac5997890f73cef932f 2024-11-19T11:15:22,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/d47362c958fa4ddb93e71b9bfe0f88f8 is 50, key is test_row_0/C:col10/1732014921102/Put/seqid=0 2024-11-19T11:15:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:22,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:22,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742087_1263 (size=12001) 2024-11-19T11:15:22,254 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/d47362c958fa4ddb93e71b9bfe0f88f8 2024-11-19T11:15:22,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/f447dfb6161b477ab0a1934de6004a3e as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/f447dfb6161b477ab0a1934de6004a3e 2024-11-19T11:15:22,263 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/f447dfb6161b477ab0a1934de6004a3e, entries=150, sequenceid=129, filesize=11.7 K 2024-11-19T11:15:22,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/cd224fc0e4dc4ac5997890f73cef932f as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/cd224fc0e4dc4ac5997890f73cef932f 2024-11-19T11:15:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T11:15:22,280 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/cd224fc0e4dc4ac5997890f73cef932f, entries=150, sequenceid=129, filesize=11.7 K 2024-11-19T11:15:22,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/d47362c958fa4ddb93e71b9bfe0f88f8 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/d47362c958fa4ddb93e71b9bfe0f88f8 2024-11-19T11:15:22,288 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/d47362c958fa4ddb93e71b9bfe0f88f8, entries=150, sequenceid=129, filesize=11.7 K 2024-11-19T11:15:22,290 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=107.34 KB/109920 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 173ms, sequenceid=129, compaction requested=true 2024-11-19T11:15:22,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:22,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:22,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-19T11:15:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-19T11:15:22,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-19T11:15:22,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 327 msec 2024-11-19T11:15:22,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732014922276, max timestamp=1732014922291Num uniques 859; 2024-11-19T11:15:22,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732014922276, max timestamp=1732014922290Num uniques 859; 2024-11-19T11:15:22,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T11:15:22,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T11:15:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:22,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732014922276, max timestamp=1732014922291Num uniques 859; 2024-11-19T11:15:22,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T11:15:22,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=40.25 KB heapSize=101.39 KB 2024-11-19T11:15:22,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 336 msec 2024-11-19T11:15:22,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:22,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:22,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:22,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:22,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:22,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:22,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/9efbb97ee5ad4fc58f972546a460bf43 is 50, key is test_row_0/A:col10/1732014922293/Put/seqid=0 2024-11-19T11:15:22,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742088_1264 (size=12151) 2024-11-19T11:15:22,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/9efbb97ee5ad4fc58f972546a460bf43 2024-11-19T11:15:22,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014922327, max timestamp=1732014922335Num uniques 859; 2024-11-19T11:15:22,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014922327, max timestamp=1732014922335Num uniques 859; 2024-11-19T11:15:22,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:15:22,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014922327, max timestamp=1732014922335Num uniques 859; 2024-11-19T11:15:22,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:15:22,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:15:22,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/9aad18f7297148ca8c9a101ae57cd767 is 50, key is test_row_0/B:col10/1732014922293/Put/seqid=0 2024-11-19T11:15:22,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742089_1265 (size=12151) 2024-11-19T11:15:22,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/9aad18f7297148ca8c9a101ae57cd767 2024-11-19T11:15:22,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/b5003d8669954ad888af670e22f9b41d is 50, key is test_row_0/C:col10/1732014922293/Put/seqid=0 2024-11-19T11:15:22,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742090_1266 (size=12151) 2024-11-19T11:15:22,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/b5003d8669954ad888af670e22f9b41d 2024-11-19T11:15:22,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/9efbb97ee5ad4fc58f972546a460bf43 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/9efbb97ee5ad4fc58f972546a460bf43 2024-11-19T11:15:22,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/9efbb97ee5ad4fc58f972546a460bf43, entries=150, sequenceid=152, filesize=11.9 K 2024-11-19T11:15:22,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/9aad18f7297148ca8c9a101ae57cd767 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/9aad18f7297148ca8c9a101ae57cd767 2024-11-19T11:15:22,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/9aad18f7297148ca8c9a101ae57cd767, entries=150, sequenceid=152, filesize=11.9 K 2024-11-19T11:15:22,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/b5003d8669954ad888af670e22f9b41d as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/b5003d8669954ad888af670e22f9b41d 2024-11-19T11:15:22,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014982415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:22,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:22,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014982417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:22,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:22,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732014922380, max timestamp=1732014922428Num uniques 1018; 2024-11-19T11:15:22,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732014922380, max timestamp=1732014922428Num uniques 1018; 2024-11-19T11:15:22,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T11:15:22,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T11:15:22,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732014922380, max timestamp=1732014922428Num uniques 1018; 2024-11-19T11:15:22,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T11:15:22,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/b5003d8669954ad888af670e22f9b41d, entries=150, sequenceid=152, filesize=11.9 K 2024-11-19T11:15:22,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~121.27 KB/124176, currentSize=46.96 KB/48090 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 147ms, sequenceid=152, compaction requested=true 2024-11-19T11:15:22,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:22,441 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:15:22,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:15:22,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:22,442 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:15:22,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:15:22,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:22,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:15:22,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:22,443 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:15:22,443 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/A is initiating minor compaction (all files) 2024-11-19T11:15:22,443 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/A in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:22,443 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/1f736e7129d54e258cbddf5a55533299, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/325ee4ecceef40bf93f338d9d6a887ff, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/f447dfb6161b477ab0a1934de6004a3e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/9efbb97ee5ad4fc58f972546a460bf43] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=47.2 K 2024-11-19T11:15:22,444 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f736e7129d54e258cbddf5a55533299, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732014920357 2024-11-19T11:15:22,445 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:15:22,445 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/B is initiating minor compaction (all files) 2024-11-19T11:15:22,445 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/B in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:22,445 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/f625d0d012894534b558f8c0387f3c24, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/128e2a3e0f294d8fa9d43606c38dbdab, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/cd224fc0e4dc4ac5997890f73cef932f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/9aad18f7297148ca8c9a101ae57cd767] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=47.2 K 2024-11-19T11:15:22,446 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 325ee4ecceef40bf93f338d9d6a887ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732014920411 2024-11-19T11:15:22,446 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting f625d0d012894534b558f8c0387f3c24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732014920357 2024-11-19T11:15:22,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=53.67 KB heapSize=140.39 KB 2024-11-19T11:15:22,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:22,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:22,446 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting f447dfb6161b477ab0a1934de6004a3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732014921093 2024-11-19T11:15:22,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:22,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:22,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:22,446 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 128e2a3e0f294d8fa9d43606c38dbdab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732014920411 2024-11-19T11:15:22,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:22,447 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9efbb97ee5ad4fc58f972546a460bf43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732014922285 2024-11-19T11:15:22,448 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting cd224fc0e4dc4ac5997890f73cef932f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732014921093 2024-11-19T11:15:22,449 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 9aad18f7297148ca8c9a101ae57cd767, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732014922285 2024-11-19T11:15:22,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/0d938065d96745a2804b6ce8772d255d is 50, key is test_row_0/A:col10/1732014922442/Put/seqid=0 2024-11-19T11:15:22,475 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#A#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:22,476 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/cb2b9f20e8714cffb0b88e957709d6d8 is 50, key is test_row_0/A:col10/1732014922293/Put/seqid=0 2024-11-19T11:15:22,498 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#B#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:22,499 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/bd626fc427864160abe94c2719beae51 is 50, key is test_row_0/B:col10/1732014922293/Put/seqid=0 2024-11-19T11:15:22,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742091_1267 (size=14541) 2024-11-19T11:15:22,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742092_1268 (size=12493) 2024-11-19T11:15:22,511 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/cb2b9f20e8714cffb0b88e957709d6d8 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/cb2b9f20e8714cffb0b88e957709d6d8 2024-11-19T11:15:22,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732014922481, max timestamp=1732014922510Num uniques 859; 2024-11-19T11:15:22,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732014922481, max timestamp=1732014922510Num uniques 859; 2024-11-19T11:15:22,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T11:15:22,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T11:15:22,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732014922481, max timestamp=1732014922510Num uniques 859; 2024-11-19T11:15:22,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T11:15:22,517 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/A of ae1beb1ad4dbb1059f68b61e2fd94bf8 into cb2b9f20e8714cffb0b88e957709d6d8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:22,517 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:22,517 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/A, priority=12, startTime=1732014922441; duration=0sec 2024-11-19T11:15:22,518 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:22,518 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:A 2024-11-19T11:15:22,518 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:15:22,519 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:15:22,519 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/C is initiating minor compaction (all files) 2024-11-19T11:15:22,519 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/C in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:22,519 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/0d3c64898aff40f2ba4fa1eefbdce613, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/ffb6c0abdde9409bb25e2f1064ea31a5, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/d47362c958fa4ddb93e71b9bfe0f88f8, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/b5003d8669954ad888af670e22f9b41d] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=47.2 K 2024-11-19T11:15:22,519 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d3c64898aff40f2ba4fa1eefbdce613, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732014920357 2024-11-19T11:15:22,520 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffb6c0abdde9409bb25e2f1064ea31a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732014920411 2024-11-19T11:15:22,520 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting d47362c958fa4ddb93e71b9bfe0f88f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732014921093 2024-11-19T11:15:22,520 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5003d8669954ad888af670e22f9b41d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732014922285 2024-11-19T11:15:22,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742093_1269 (size=12493) 2024-11-19T11:15:22,544 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#C#compaction#227 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:22,544 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/819ec6fca0874da4ad2c4322580c00d8 is 50, key is test_row_0/C:col10/1732014922293/Put/seqid=0 2024-11-19T11:15:22,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014982552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:22,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014982552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:22,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014982554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:22,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014982555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:22,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014982556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:22,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742094_1270 (size=12493) 2024-11-19T11:15:22,579 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/819ec6fca0874da4ad2c4322580c00d8 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/819ec6fca0874da4ad2c4322580c00d8 2024-11-19T11:15:22,587 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/C of ae1beb1ad4dbb1059f68b61e2fd94bf8 into 819ec6fca0874da4ad2c4322580c00d8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:22,587 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:22,587 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/C, priority=12, startTime=1732014922442; duration=0sec 2024-11-19T11:15:22,587 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:22,587 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:C 2024-11-19T11:15:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T11:15:22,588 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:15:22,588 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:15:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-19T11:15:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T11:15:22,590 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:15:22,591 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:15:22,591 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:15:22,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014982668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:22,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:22,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014982668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:22,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:22,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014982668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:22,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:22,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014982669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:22,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:22,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014982669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:22,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T11:15:22,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T11:15:22,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:22,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:22,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:22,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:22,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:22,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014982877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:22,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014982877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:22,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014982877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:22,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014982877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:22,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:22,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:22,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014982877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:22,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:22,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:22,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:22,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T11:15:22,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:22,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:22,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:22,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:22,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:22,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/0d938065d96745a2804b6ce8772d255d 2024-11-19T11:15:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T11:15:22,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/2ff70e6af620454da4c721aa87fac722 is 50, key is test_row_0/B:col10/1732014922442/Put/seqid=0 2024-11-19T11:15:22,944 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/bd626fc427864160abe94c2719beae51 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/bd626fc427864160abe94c2719beae51 2024-11-19T11:15:22,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742095_1271 (size=12151) 2024-11-19T11:15:22,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/2ff70e6af620454da4c721aa87fac722 2024-11-19T11:15:22,959 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/B of ae1beb1ad4dbb1059f68b61e2fd94bf8 into bd626fc427864160abe94c2719beae51(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:22,959 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:22,959 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/B, priority=12, startTime=1732014922442; duration=0sec 2024-11-19T11:15:22,959 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:22,959 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:B 2024-11-19T11:15:22,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/4c910a28d756430faa4d824a937521a2 is 50, key is test_row_0/C:col10/1732014922442/Put/seqid=0 2024-11-19T11:15:22,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742096_1272 (size=12151) 2024-11-19T11:15:23,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T11:15:23,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:23,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:23,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:23,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:23,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:23,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014983187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:23,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:23,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014983187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:23,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:23,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014983187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:23,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014983188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:23,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014983188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:23,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T11:15:23,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:23,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:23,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:23,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:23,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T11:15:23,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T11:15:23,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:23,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:23,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:23,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:23,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:23,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/4c910a28d756430faa4d824a937521a2 2024-11-19T11:15:23,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/0d938065d96745a2804b6ce8772d255d as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/0d938065d96745a2804b6ce8772d255d 2024-11-19T11:15:23,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/0d938065d96745a2804b6ce8772d255d, entries=200, sequenceid=193, filesize=14.2 K 2024-11-19T11:15:23,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/2ff70e6af620454da4c721aa87fac722 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/2ff70e6af620454da4c721aa87fac722 2024-11-19T11:15:23,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/2ff70e6af620454da4c721aa87fac722, entries=150, sequenceid=193, filesize=11.9 K 2024-11-19T11:15:23,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/4c910a28d756430faa4d824a937521a2 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/4c910a28d756430faa4d824a937521a2 2024-11-19T11:15:23,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/4c910a28d756430faa4d824a937521a2, entries=150, sequenceid=193, filesize=11.9 K 2024-11-19T11:15:23,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~140.34 KB/143712, currentSize=134.18 KB/137400 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 964ms, sequenceid=193, compaction requested=false 2024-11-19T11:15:23,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:23,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T11:15:23,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:23,511 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-19T11:15:23,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:23,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:23,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:23,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:23,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:23,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:23,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/64d12c4973184096902714c2d1fad516 is 50, key is test_row_0/A:col10/1732014922553/Put/seqid=0 2024-11-19T11:15:23,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742097_1273 (size=12151) 2024-11-19T11:15:23,533 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/64d12c4973184096902714c2d1fad516 2024-11-19T11:15:23,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/fbae115eef9f488d805a6136c1855bd4 is 50, key is test_row_0/B:col10/1732014922553/Put/seqid=0 2024-11-19T11:15:23,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742098_1274 (size=12151) 2024-11-19T11:15:23,559 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/fbae115eef9f488d805a6136c1855bd4 2024-11-19T11:15:23,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/91734aab1f404fb2af836c067b54e6c7 is 50, key is test_row_0/C:col10/1732014922553/Put/seqid=0 2024-11-19T11:15:23,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742099_1275 (size=12151) 2024-11-19T11:15:23,581 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/91734aab1f404fb2af836c067b54e6c7 2024-11-19T11:15:23,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/64d12c4973184096902714c2d1fad516 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/64d12c4973184096902714c2d1fad516 2024-11-19T11:15:23,594 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/64d12c4973184096902714c2d1fad516, entries=150, sequenceid=231, filesize=11.9 K 2024-11-19T11:15:23,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/fbae115eef9f488d805a6136c1855bd4 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/fbae115eef9f488d805a6136c1855bd4 2024-11-19T11:15:23,598 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/fbae115eef9f488d805a6136c1855bd4, entries=150, sequenceid=231, filesize=11.9 K 2024-11-19T11:15:23,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/91734aab1f404fb2af836c067b54e6c7 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/91734aab1f404fb2af836c067b54e6c7 2024-11-19T11:15:23,609 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/91734aab1f404fb2af836c067b54e6c7, entries=150, sequenceid=231, filesize=11.9 K 2024-11-19T11:15:23,610 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=0 B/0 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 99ms, sequenceid=231, compaction requested=true 2024-11-19T11:15:23,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:23,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:23,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-19T11:15:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-19T11:15:23,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-19T11:15:23,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0210 sec 2024-11-19T11:15:23,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.0270 sec 2024-11-19T11:15:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:23,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T11:15:23,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:23,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:23,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:23,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:23,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:23,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:23,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/424a7d04f3b04298b45bff5226366087 is 50, key is test_row_0/A:col10/1732014923699/Put/seqid=0 2024-11-19T11:15:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T11:15:23,727 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:15:23,727 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:15:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-19T11:15:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T11:15:23,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732014923709, max timestamp=1732014923728Num uniques 859; 2024-11-19T11:15:23,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732014923709, max timestamp=1732014923728Num uniques 859; 2024-11-19T11:15:23,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732014923709, max timestamp=1732014923728Num uniques 859; 2024-11-19T11:15:23,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T11:15:23,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T11:15:23,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T11:15:23,734 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:15:23,735 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:15:23,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742100_1276 (size=14541) 2024-11-19T11:15:23,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:15:23,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/424a7d04f3b04298b45bff5226366087 2024-11-19T11:15:23,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/2809b4cd197c4950b2cbc46bbefe0666 is 50, key is test_row_0/B:col10/1732014923699/Put/seqid=0 2024-11-19T11:15:23,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:23,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014983754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014983753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:23,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742101_1277 (size=12151) 2024-11-19T11:15:23,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/2809b4cd197c4950b2cbc46bbefe0666 2024-11-19T11:15:23,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:23,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014983761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014983760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:23,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014983780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:23,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:23,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:23,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/ec1fcef53b16414b81fbc36dc46b3630 is 50, key is test_row_0/C:col10/1732014923699/Put/seqid=0 2024-11-19T11:15:23,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742102_1278 (size=12151) 2024-11-19T11:15:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T11:15:23,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T11:15:23,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:23,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:23,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:23,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:23,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:23,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014983888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:23,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:23,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:23,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014983888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:23,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:23,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014983888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:23,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:23,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014983888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:23,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:23,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014983889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:23,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:23,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:23,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:24,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T11:15:24,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:24,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:24,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:24,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:24,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T11:15:24,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014984097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:24,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014984097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:24,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014984097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:24,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014984098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:24,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014984107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:24,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/ec1fcef53b16414b81fbc36dc46b3630 2024-11-19T11:15:24,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/424a7d04f3b04298b45bff5226366087 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/424a7d04f3b04298b45bff5226366087 2024-11-19T11:15:24,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/424a7d04f3b04298b45bff5226366087, entries=200, sequenceid=241, filesize=14.2 K 2024-11-19T11:15:24,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/2809b4cd197c4950b2cbc46bbefe0666 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/2809b4cd197c4950b2cbc46bbefe0666 2024-11-19T11:15:24,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/2809b4cd197c4950b2cbc46bbefe0666, entries=150, sequenceid=241, filesize=11.9 K 2024-11-19T11:15:24,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/ec1fcef53b16414b81fbc36dc46b3630 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/ec1fcef53b16414b81fbc36dc46b3630 2024-11-19T11:15:24,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T11:15:24,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:24,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:24,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:24,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:24,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:24,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:24,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/ec1fcef53b16414b81fbc36dc46b3630, entries=150, sequenceid=241, filesize=11.9 K 2024-11-19T11:15:24,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 519ms, sequenceid=241, compaction requested=true 2024-11-19T11:15:24,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:24,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:15:24,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:24,222 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:15:24,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:15:24,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:24,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:15:24,222 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:15:24,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:24,223 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:15:24,223 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:15:24,224 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/B is initiating minor compaction (all files) 2024-11-19T11:15:24,224 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/A is initiating minor compaction (all files) 2024-11-19T11:15:24,224 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/A in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:24,224 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/B in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:24,224 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/bd626fc427864160abe94c2719beae51, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/2ff70e6af620454da4c721aa87fac722, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/fbae115eef9f488d805a6136c1855bd4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/2809b4cd197c4950b2cbc46bbefe0666] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=47.8 K 2024-11-19T11:15:24,224 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/cb2b9f20e8714cffb0b88e957709d6d8, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/0d938065d96745a2804b6ce8772d255d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/64d12c4973184096902714c2d1fad516, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/424a7d04f3b04298b45bff5226366087] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=52.5 K 2024-11-19T11:15:24,224 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting bd626fc427864160abe94c2719beae51, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732014922285 2024-11-19T11:15:24,224 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb2b9f20e8714cffb0b88e957709d6d8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732014922285 2024-11-19T11:15:24,224 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ff70e6af620454da4c721aa87fac722, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732014922428 2024-11-19T11:15:24,224 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d938065d96745a2804b6ce8772d255d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732014922428 2024-11-19T11:15:24,225 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting fbae115eef9f488d805a6136c1855bd4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732014922532 2024-11-19T11:15:24,225 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64d12c4973184096902714c2d1fad516, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732014922532 2024-11-19T11:15:24,225 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 2809b4cd197c4950b2cbc46bbefe0666, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732014923699 2024-11-19T11:15:24,225 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 424a7d04f3b04298b45bff5226366087, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732014923699 2024-11-19T11:15:24,233 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#A#compaction#236 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:24,234 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#B#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:24,234 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/1cc921af62c642eb9f35c8a2406fddfd is 50, key is test_row_0/A:col10/1732014923699/Put/seqid=0 2024-11-19T11:15:24,235 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/e48c7e82ffac4a0b8c9cfbaec3221dd2 is 50, key is test_row_0/B:col10/1732014923699/Put/seqid=0 2024-11-19T11:15:24,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742104_1280 (size=12629) 2024-11-19T11:15:24,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742103_1279 (size=12629) 2024-11-19T11:15:24,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T11:15:24,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T11:15:24,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:24,372 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-19T11:15:24,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:24,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:24,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:24,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:24,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:24,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:24,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/cc1d2393bbb046a3a0d808214a4872ac is 50, key is test_row_0/A:col10/1732014923759/Put/seqid=0 2024-11-19T11:15:24,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742105_1281 (size=12301) 2024-11-19T11:15:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:24,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:24,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014984418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:24,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014984418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:24,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014984418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:24,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:24,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014984439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:24,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014984439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:24,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014984548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:24,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:24,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014984548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:24,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:24,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014984548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:24,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014984548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:24,650 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/e48c7e82ffac4a0b8c9cfbaec3221dd2 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/e48c7e82ffac4a0b8c9cfbaec3221dd2 2024-11-19T11:15:24,659 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/1cc921af62c642eb9f35c8a2406fddfd as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/1cc921af62c642eb9f35c8a2406fddfd 2024-11-19T11:15:24,662 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/B of ae1beb1ad4dbb1059f68b61e2fd94bf8 into e48c7e82ffac4a0b8c9cfbaec3221dd2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:24,662 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:24,662 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/B, priority=12, startTime=1732014924222; duration=0sec 2024-11-19T11:15:24,663 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:24,663 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:B 2024-11-19T11:15:24,663 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:15:24,664 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:15:24,665 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/C is initiating minor compaction (all files) 2024-11-19T11:15:24,665 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/C in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:24,665 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/819ec6fca0874da4ad2c4322580c00d8, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/4c910a28d756430faa4d824a937521a2, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/91734aab1f404fb2af836c067b54e6c7, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/ec1fcef53b16414b81fbc36dc46b3630] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=47.8 K 2024-11-19T11:15:24,667 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 819ec6fca0874da4ad2c4322580c00d8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732014922285 2024-11-19T11:15:24,667 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c910a28d756430faa4d824a937521a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732014922428 2024-11-19T11:15:24,669 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 91734aab1f404fb2af836c067b54e6c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732014922532 2024-11-19T11:15:24,670 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting ec1fcef53b16414b81fbc36dc46b3630, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732014923699 2024-11-19T11:15:24,671 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/A of ae1beb1ad4dbb1059f68b61e2fd94bf8 into 1cc921af62c642eb9f35c8a2406fddfd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:24,671 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:24,671 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/A, priority=12, startTime=1732014924222; duration=0sec 2024-11-19T11:15:24,671 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:24,672 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:A 2024-11-19T11:15:24,688 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#C#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:24,694 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/5bcee6e80d294763bfaa6d05b0aaef14 is 50, key is test_row_0/C:col10/1732014923699/Put/seqid=0 2024-11-19T11:15:24,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742106_1282 (size=12629) 2024-11-19T11:15:24,759 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T11:15:24,759 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T11:15:24,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014984757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:24,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014984757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:24,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014984757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:24,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014984757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:24,804 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/cc1d2393bbb046a3a0d808214a4872ac 2024-11-19T11:15:24,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/c622c2f2c95c4b8bb28a83974645f0fa is 50, key is test_row_0/B:col10/1732014923759/Put/seqid=0 2024-11-19T11:15:24,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742107_1283 (size=12301) 2024-11-19T11:15:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T11:15:24,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014984948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:24,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:24,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:25,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:25,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014985068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014985068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:25,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:25,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014985068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014985068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:25,124 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/5bcee6e80d294763bfaa6d05b0aaef14 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/5bcee6e80d294763bfaa6d05b0aaef14 2024-11-19T11:15:25,130 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/C of ae1beb1ad4dbb1059f68b61e2fd94bf8 into 5bcee6e80d294763bfaa6d05b0aaef14(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:25,130 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:25,130 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/C, priority=12, startTime=1732014924222; duration=0sec 2024-11-19T11:15:25,130 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:25,130 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:C 2024-11-19T11:15:25,230 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/c622c2f2c95c4b8bb28a83974645f0fa 2024-11-19T11:15:25,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/a1f35b9ab82648e9bd792f6c1570c6ca is 50, key is test_row_0/C:col10/1732014923759/Put/seqid=0 2024-11-19T11:15:25,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742108_1284 (size=12301) 2024-11-19T11:15:25,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014985578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:25,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:25,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014985578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014985578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:25,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014985578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:25,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:25,644 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/a1f35b9ab82648e9bd792f6c1570c6ca 2024-11-19T11:15:25,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/cc1d2393bbb046a3a0d808214a4872ac as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/cc1d2393bbb046a3a0d808214a4872ac 2024-11-19T11:15:25,652 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/cc1d2393bbb046a3a0d808214a4872ac, entries=150, sequenceid=275, filesize=12.0 K 2024-11-19T11:15:25,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/c622c2f2c95c4b8bb28a83974645f0fa as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/c622c2f2c95c4b8bb28a83974645f0fa 2024-11-19T11:15:25,656 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/c622c2f2c95c4b8bb28a83974645f0fa, entries=150, sequenceid=275, filesize=12.0 K 2024-11-19T11:15:25,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/a1f35b9ab82648e9bd792f6c1570c6ca as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/a1f35b9ab82648e9bd792f6c1570c6ca 2024-11-19T11:15:25,662 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/a1f35b9ab82648e9bd792f6c1570c6ca, entries=150, sequenceid=275, filesize=12.0 K 2024-11-19T11:15:25,663 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=53.67 KB/54960 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 1291ms, sequenceid=275, compaction requested=false 2024-11-19T11:15:25,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:25,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:25,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-19T11:15:25,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-19T11:15:25,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-19T11:15:25,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9340 sec 2024-11-19T11:15:25,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.9450 sec 2024-11-19T11:15:25,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T11:15:25,878 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:15:25,878 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:15:25,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-19T11:15:25,880 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:15:25,880 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:15:25,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:15:25,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T11:15:25,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:25,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T11:15:25,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:25,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:25,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:25,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:25,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:25,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:25,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/4f8b0f56241840f68f920fb440717211 is 50, key is test_row_0/A:col10/1732014925958/Put/seqid=0 2024-11-19T11:15:25,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742109_1285 (size=14741) 2024-11-19T11:15:25,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/4f8b0f56241840f68f920fb440717211 2024-11-19T11:15:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T11:15:25,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/ce62572c1a324c94a11b5fbf76ada02d is 50, key is test_row_0/B:col10/1732014925958/Put/seqid=0 2024-11-19T11:15:26,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742110_1286 (size=12301) 2024-11-19T11:15:26,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/ce62572c1a324c94a11b5fbf76ada02d 2024-11-19T11:15:26,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/902b3ed4792d4ab2abce526b5847bd83 is 50, key is test_row_0/C:col10/1732014925958/Put/seqid=0 2024-11-19T11:15:26,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T11:15:26,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:26,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:26,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:26,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:26,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:26,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:26,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742111_1287 (size=12301) 2024-11-19T11:15:26,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014986070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:26,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:26,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014986178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:26,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:26,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T11:15:26,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:26,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:26,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:26,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:26,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T11:15:26,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T11:15:26,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:26,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:26,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:26,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:26,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:26,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014986388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:26,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:26,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/902b3ed4792d4ab2abce526b5847bd83 2024-11-19T11:15:26,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/4f8b0f56241840f68f920fb440717211 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/4f8b0f56241840f68f920fb440717211 2024-11-19T11:15:26,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/4f8b0f56241840f68f920fb440717211, entries=200, sequenceid=290, filesize=14.4 K 2024-11-19T11:15:26,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/ce62572c1a324c94a11b5fbf76ada02d as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/ce62572c1a324c94a11b5fbf76ada02d 2024-11-19T11:15:26,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/ce62572c1a324c94a11b5fbf76ada02d, entries=150, sequenceid=290, filesize=12.0 K 2024-11-19T11:15:26,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/902b3ed4792d4ab2abce526b5847bd83 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/902b3ed4792d4ab2abce526b5847bd83 2024-11-19T11:15:26,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/902b3ed4792d4ab2abce526b5847bd83, entries=150, sequenceid=290, filesize=12.0 K 2024-11-19T11:15:26,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 499ms, sequenceid=290, compaction requested=true 2024-11-19T11:15:26,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:26,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:15:26,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:26,460 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:26,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:15:26,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:26,460 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:26,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:15:26,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:26,462 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:26,462 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/B is initiating minor compaction (all files) 2024-11-19T11:15:26,462 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/B in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:26,462 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/e48c7e82ffac4a0b8c9cfbaec3221dd2, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/c622c2f2c95c4b8bb28a83974645f0fa, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/ce62572c1a324c94a11b5fbf76ada02d] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=36.4 K 2024-11-19T11:15:26,462 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39671 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:26,462 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/A is initiating minor compaction (all files) 2024-11-19T11:15:26,462 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/A in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:26,462 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/1cc921af62c642eb9f35c8a2406fddfd, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/cc1d2393bbb046a3a0d808214a4872ac, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/4f8b0f56241840f68f920fb440717211] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=38.7 K 2024-11-19T11:15:26,462 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting e48c7e82ffac4a0b8c9cfbaec3221dd2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732014923699 2024-11-19T11:15:26,463 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cc921af62c642eb9f35c8a2406fddfd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732014923699 2024-11-19T11:15:26,463 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting c622c2f2c95c4b8bb28a83974645f0fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732014923751 2024-11-19T11:15:26,463 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc1d2393bbb046a3a0d808214a4872ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732014923751 2024-11-19T11:15:26,463 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting ce62572c1a324c94a11b5fbf76ada02d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732014924412 2024-11-19T11:15:26,463 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f8b0f56241840f68f920fb440717211, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732014924412 2024-11-19T11:15:26,474 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#A#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:26,474 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/fae6b89d02a641538b684cf2b4ad0b5f is 50, key is test_row_0/A:col10/1732014925958/Put/seqid=0 2024-11-19T11:15:26,477 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#B#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:26,478 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/2673b9e8a2664a15b70fb48ac8873492 is 50, key is test_row_0/B:col10/1732014925958/Put/seqid=0 2024-11-19T11:15:26,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742113_1289 (size=12881) 2024-11-19T11:15:26,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742112_1288 (size=12881) 2024-11-19T11:15:26,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T11:15:26,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:26,512 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T11:15:26,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:26,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:26,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:26,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:26,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:26,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:26,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/216c4d395da847c7ac2378ebac0ba5ad is 50, key is test_row_0/A:col10/1732014926059/Put/seqid=0 2024-11-19T11:15:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T11:15:26,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742114_1290 (size=12301) 2024-11-19T11:15:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:26,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:26,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014986609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014986609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:26,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014986613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:26,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014986614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:26,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014986697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:26,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:26,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014986717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:26,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014986718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:26,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014986718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:26,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014986718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:26,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:26,903 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/2673b9e8a2664a15b70fb48ac8873492 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/2673b9e8a2664a15b70fb48ac8873492 2024-11-19T11:15:26,907 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/fae6b89d02a641538b684cf2b4ad0b5f as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/fae6b89d02a641538b684cf2b4ad0b5f 2024-11-19T11:15:26,908 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/B of ae1beb1ad4dbb1059f68b61e2fd94bf8 into 2673b9e8a2664a15b70fb48ac8873492(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:26,908 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:26,908 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/B, priority=13, startTime=1732014926460; duration=0sec 2024-11-19T11:15:26,909 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:26,909 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:B 2024-11-19T11:15:26,909 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:26,911 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:26,911 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/C is initiating minor compaction (all files) 2024-11-19T11:15:26,911 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/C in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:26,911 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/5bcee6e80d294763bfaa6d05b0aaef14, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/a1f35b9ab82648e9bd792f6c1570c6ca, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/902b3ed4792d4ab2abce526b5847bd83] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=36.4 K 2024-11-19T11:15:26,913 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bcee6e80d294763bfaa6d05b0aaef14, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732014923699 2024-11-19T11:15:26,913 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting a1f35b9ab82648e9bd792f6c1570c6ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732014923751 2024-11-19T11:15:26,914 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/A of ae1beb1ad4dbb1059f68b61e2fd94bf8 into fae6b89d02a641538b684cf2b4ad0b5f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:26,914 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:26,914 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/A, priority=13, startTime=1732014926460; duration=0sec 2024-11-19T11:15:26,915 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 902b3ed4792d4ab2abce526b5847bd83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732014924412 2024-11-19T11:15:26,915 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:26,915 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:A 2024-11-19T11:15:26,920 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/216c4d395da847c7ac2378ebac0ba5ad 2024-11-19T11:15:26,922 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#C#compaction#248 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:26,922 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/6e80057f18fb46599418627ee47094e0 is 50, key is test_row_0/C:col10/1732014925958/Put/seqid=0 2024-11-19T11:15:26,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014986928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:26,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:26,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:26,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014986928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014986928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:26,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014986929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:26,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/abb798a2a19043daae7ab1afec337a61 is 50, key is test_row_0/B:col10/1732014926059/Put/seqid=0 2024-11-19T11:15:26,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742115_1291 (size=12881) 2024-11-19T11:15:26,945 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/6e80057f18fb46599418627ee47094e0 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/6e80057f18fb46599418627ee47094e0 2024-11-19T11:15:26,953 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/C of ae1beb1ad4dbb1059f68b61e2fd94bf8 into 6e80057f18fb46599418627ee47094e0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:26,953 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:26,953 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/C, priority=13, startTime=1732014926460; duration=0sec 2024-11-19T11:15:26,953 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:26,953 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:C 2024-11-19T11:15:26,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742116_1292 (size=12301) 2024-11-19T11:15:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T11:15:27,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014987207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:27,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:27,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:27,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:27,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014987238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:27,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:27,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:27,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:27,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014987238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:27,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:27,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:27,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:27,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014987238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:27,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:27,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:27,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:27,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014987238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:27,358 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/abb798a2a19043daae7ab1afec337a61 2024-11-19T11:15:27,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/50981b1937fb49d3a748fbe44c5f493a is 50, key is test_row_0/C:col10/1732014926059/Put/seqid=0 2024-11-19T11:15:27,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742117_1293 (size=12301) 2024-11-19T11:15:27,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014987748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:27,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:27,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:27,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:27,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:27,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014987757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:27,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:27,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:27,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:27,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014987757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:27,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:27,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:27,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014987757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:27,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:27,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:27,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:27,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:27,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:27,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:27,779 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/50981b1937fb49d3a748fbe44c5f493a 2024-11-19T11:15:27,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/216c4d395da847c7ac2378ebac0ba5ad as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/216c4d395da847c7ac2378ebac0ba5ad 2024-11-19T11:15:27,786 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/216c4d395da847c7ac2378ebac0ba5ad, entries=150, sequenceid=310, filesize=12.0 K 2024-11-19T11:15:27,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/abb798a2a19043daae7ab1afec337a61 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/abb798a2a19043daae7ab1afec337a61 2024-11-19T11:15:27,791 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/abb798a2a19043daae7ab1afec337a61, entries=150, sequenceid=310, filesize=12.0 K 2024-11-19T11:15:27,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/50981b1937fb49d3a748fbe44c5f493a as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/50981b1937fb49d3a748fbe44c5f493a 2024-11-19T11:15:27,797 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/50981b1937fb49d3a748fbe44c5f493a, entries=150, sequenceid=310, filesize=12.0 K 2024-11-19T11:15:27,798 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 1286ms, sequenceid=310, compaction requested=false 2024-11-19T11:15:27,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:27,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:27,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-19T11:15:27,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-19T11:15:27,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-19T11:15:27,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9190 sec 2024-11-19T11:15:27,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.9240 sec 2024-11-19T11:15:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T11:15:28,038 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:15:28,038 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:15:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-19T11:15:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T11:15:28,040 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:15:28,041 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:15:28,041 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:15:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T11:15:28,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T11:15:28,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:28,194 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T11:15:28,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:28,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:28,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:28,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:28,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:28,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:28,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/b9cb5ba95b3743cfbc99c5d356b969d2 is 50, key is test_row_0/A:col10/1732014926606/Put/seqid=0 2024-11-19T11:15:28,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742118_1294 (size=12301) 2024-11-19T11:15:28,214 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/b9cb5ba95b3743cfbc99c5d356b969d2 2024-11-19T11:15:28,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/e81c1f0622e445e2ba3c2f4c11a8c51c is 50, key is test_row_0/B:col10/1732014926606/Put/seqid=0 2024-11-19T11:15:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:28,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:28,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742119_1295 (size=12301) 2024-11-19T11:15:28,231 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/e81c1f0622e445e2ba3c2f4c11a8c51c 2024-11-19T11:15:28,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/e7b6c65861ac4cf8ae324229170788bc is 50, key is test_row_0/C:col10/1732014926606/Put/seqid=0 2024-11-19T11:15:28,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742120_1296 (size=12301) 2024-11-19T11:15:28,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014988334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:28,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T11:15:28,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014988448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:28,642 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/e7b6c65861ac4cf8ae324229170788bc 2024-11-19T11:15:28,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/b9cb5ba95b3743cfbc99c5d356b969d2 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/b9cb5ba95b3743cfbc99c5d356b969d2 2024-11-19T11:15:28,652 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/b9cb5ba95b3743cfbc99c5d356b969d2, entries=150, sequenceid=326, filesize=12.0 K 2024-11-19T11:15:28,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/e81c1f0622e445e2ba3c2f4c11a8c51c as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/e81c1f0622e445e2ba3c2f4c11a8c51c 2024-11-19T11:15:28,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014988658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:28,664 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/e81c1f0622e445e2ba3c2f4c11a8c51c, entries=150, sequenceid=326, filesize=12.0 K 2024-11-19T11:15:28,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/e7b6c65861ac4cf8ae324229170788bc as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/e7b6c65861ac4cf8ae324229170788bc 2024-11-19T11:15:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T11:15:28,672 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/e7b6c65861ac4cf8ae324229170788bc, entries=150, sequenceid=326, filesize=12.0 K 2024-11-19T11:15:28,673 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 479ms, sequenceid=326, compaction requested=true 2024-11-19T11:15:28,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:28,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:28,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-19T11:15:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-19T11:15:28,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-19T11:15:28,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 634 msec 2024-11-19T11:15:28,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 639 msec 2024-11-19T11:15:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:28,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T11:15:28,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:28,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:28,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:28,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:28,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:28,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:28,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/f70008e8e6f445849631145c6e4e538d is 50, key is test_row_0/A:col10/1732014928331/Put/seqid=0 2024-11-19T11:15:28,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742121_1297 (size=12301) 2024-11-19T11:15:28,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014988780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:28,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014988782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:28,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014988782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:28,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014988783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:28,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014988887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:28,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:28,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:28,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014988897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:28,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:28,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:28,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014988897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:28,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:28,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:28,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014988897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:28,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:28,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:28,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014988967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:29,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014989097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:29,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:29,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014989108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:29,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:29,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014989108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:29,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:29,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014989108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:29,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:29,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/f70008e8e6f445849631145c6e4e538d 2024-11-19T11:15:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T11:15:29,179 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:15:29,179 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:15:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-19T11:15:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T11:15:29,182 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:15:29,183 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:15:29,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:15:29,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/0dd5e2106fca4e60808aa478a43466c0 is 50, key is test_row_0/B:col10/1732014928331/Put/seqid=0 2024-11-19T11:15:29,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742122_1298 (size=12301) 2024-11-19T11:15:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T11:15:29,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T11:15:29,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:29,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:29,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:29,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:29,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:29,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014989407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:29,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:29,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014989417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:29,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:29,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014989417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:29,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:29,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014989417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:29,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:29,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014989477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:29,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T11:15:29,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:29,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:29,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:29,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:29,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T11:15:29,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/0dd5e2106fca4e60808aa478a43466c0 2024-11-19T11:15:29,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/d43f8475b4464af6925556617b9894ed is 50, key is test_row_0/C:col10/1732014928331/Put/seqid=0 2024-11-19T11:15:29,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742123_1299 (size=12301) 2024-11-19T11:15:29,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T11:15:29,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:29,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:29,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:29,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:29,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:29,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T11:15:29,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:29,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:29,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:29,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:29,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T11:15:29,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014989917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:29,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:29,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014989927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:29,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:29,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:29,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014989927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:29,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:29,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014989928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:29,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:29,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:29,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:29,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T11:15:29,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:29,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:29,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:29,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:29,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:30,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/d43f8475b4464af6925556617b9894ed 2024-11-19T11:15:30,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/f70008e8e6f445849631145c6e4e538d as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/f70008e8e6f445849631145c6e4e538d 2024-11-19T11:15:30,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/f70008e8e6f445849631145c6e4e538d, entries=150, sequenceid=346, filesize=12.0 K 2024-11-19T11:15:30,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/0dd5e2106fca4e60808aa478a43466c0 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/0dd5e2106fca4e60808aa478a43466c0 2024-11-19T11:15:30,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/0dd5e2106fca4e60808aa478a43466c0, entries=150, sequenceid=346, filesize=12.0 K 2024-11-19T11:15:30,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/d43f8475b4464af6925556617b9894ed as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/d43f8475b4464af6925556617b9894ed 2024-11-19T11:15:30,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/d43f8475b4464af6925556617b9894ed, entries=150, sequenceid=346, filesize=12.0 K 2024-11-19T11:15:30,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 1288ms, sequenceid=346, compaction requested=true 2024-11-19T11:15:30,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:30,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:15:30,049 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:15:30,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:30,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:15:30,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:30,050 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:15:30,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:15:30,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:30,051 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:15:30,051 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:15:30,051 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/A is initiating minor compaction (all files) 2024-11-19T11:15:30,051 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/B is initiating minor compaction (all files) 2024-11-19T11:15:30,051 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/A in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:30,051 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/B in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:30,051 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/fae6b89d02a641538b684cf2b4ad0b5f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/216c4d395da847c7ac2378ebac0ba5ad, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/b9cb5ba95b3743cfbc99c5d356b969d2, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/f70008e8e6f445849631145c6e4e538d] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=48.6 K 2024-11-19T11:15:30,051 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/2673b9e8a2664a15b70fb48ac8873492, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/abb798a2a19043daae7ab1afec337a61, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/e81c1f0622e445e2ba3c2f4c11a8c51c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/0dd5e2106fca4e60808aa478a43466c0] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=48.6 K 2024-11-19T11:15:30,051 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting fae6b89d02a641538b684cf2b4ad0b5f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732014924412 2024-11-19T11:15:30,051 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 2673b9e8a2664a15b70fb48ac8873492, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732014924412 2024-11-19T11:15:30,051 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 216c4d395da847c7ac2378ebac0ba5ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732014926044 2024-11-19T11:15:30,051 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting abb798a2a19043daae7ab1afec337a61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732014926044 2024-11-19T11:15:30,052 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting e81c1f0622e445e2ba3c2f4c11a8c51c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732014926606 2024-11-19T11:15:30,052 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9cb5ba95b3743cfbc99c5d356b969d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732014926606 2024-11-19T11:15:30,052 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dd5e2106fca4e60808aa478a43466c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732014928325 2024-11-19T11:15:30,052 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting f70008e8e6f445849631145c6e4e538d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732014928325 2024-11-19T11:15:30,063 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#B#compaction#257 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:30,063 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#A#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:30,063 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/c9613d59cc854ffd8161ca83c0098ea1 is 50, key is test_row_0/A:col10/1732014928331/Put/seqid=0 2024-11-19T11:15:30,063 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/f88bbef4b0d74e1187d2a8cf87d75217 is 50, key is test_row_0/B:col10/1732014928331/Put/seqid=0 2024-11-19T11:15:30,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742125_1301 (size=13017) 2024-11-19T11:15:30,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742124_1300 (size=13017) 2024-11-19T11:15:30,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T11:15:30,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:30,101 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T11:15:30,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:30,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:30,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:30,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:30,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:30,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:30,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/1807f855f3144bc88a2a4b990480a182 is 50, key is test_row_0/A:col10/1732014928780/Put/seqid=0 2024-11-19T11:15:30,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742126_1302 (size=12301) 2024-11-19T11:15:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T11:15:30,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:30,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:30,492 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/c9613d59cc854ffd8161ca83c0098ea1 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/c9613d59cc854ffd8161ca83c0098ea1 2024-11-19T11:15:30,492 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/f88bbef4b0d74e1187d2a8cf87d75217 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/f88bbef4b0d74e1187d2a8cf87d75217 2024-11-19T11:15:30,497 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/B of ae1beb1ad4dbb1059f68b61e2fd94bf8 into f88bbef4b0d74e1187d2a8cf87d75217(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:30,497 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:30,498 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/B, priority=12, startTime=1732014930049; duration=0sec 2024-11-19T11:15:30,498 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:30,498 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:B 2024-11-19T11:15:30,498 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:15:30,498 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/A of ae1beb1ad4dbb1059f68b61e2fd94bf8 into c9613d59cc854ffd8161ca83c0098ea1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:30,499 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:30,499 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/A, priority=12, startTime=1732014930049; duration=0sec 2024-11-19T11:15:30,499 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:30,499 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:A 2024-11-19T11:15:30,500 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:15:30,500 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/C is initiating minor compaction (all files) 2024-11-19T11:15:30,500 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/C in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:30,500 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/6e80057f18fb46599418627ee47094e0, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/50981b1937fb49d3a748fbe44c5f493a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/e7b6c65861ac4cf8ae324229170788bc, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/d43f8475b4464af6925556617b9894ed] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=48.6 K 2024-11-19T11:15:30,500 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e80057f18fb46599418627ee47094e0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732014924412 2024-11-19T11:15:30,500 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 50981b1937fb49d3a748fbe44c5f493a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732014926044 2024-11-19T11:15:30,501 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting e7b6c65861ac4cf8ae324229170788bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732014926606 2024-11-19T11:15:30,501 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting d43f8475b4464af6925556617b9894ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732014928325 2024-11-19T11:15:30,508 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/1807f855f3144bc88a2a4b990480a182 2024-11-19T11:15:30,510 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#C#compaction#260 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:30,511 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/9cf6cbc551c3489b95a323b29ac308ec is 50, key is test_row_0/C:col10/1732014928331/Put/seqid=0 2024-11-19T11:15:30,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/d6190f5502c74856aca7f1ac7645fd2a is 50, key is test_row_0/B:col10/1732014928780/Put/seqid=0 2024-11-19T11:15:30,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742127_1303 (size=13017) 2024-11-19T11:15:30,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742128_1304 (size=12301) 2024-11-19T11:15:30,527 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/d6190f5502c74856aca7f1ac7645fd2a 2024-11-19T11:15:30,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/f3a8dde940564e38b98061861662ebc8 is 50, key is test_row_0/C:col10/1732014928780/Put/seqid=0 2024-11-19T11:15:30,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742129_1305 (size=12301) 2024-11-19T11:15:30,546 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/f3a8dde940564e38b98061861662ebc8 2024-11-19T11:15:30,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/1807f855f3144bc88a2a4b990480a182 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/1807f855f3144bc88a2a4b990480a182 2024-11-19T11:15:30,557 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/1807f855f3144bc88a2a4b990480a182, entries=150, sequenceid=358, filesize=12.0 K 2024-11-19T11:15:30,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/d6190f5502c74856aca7f1ac7645fd2a as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/d6190f5502c74856aca7f1ac7645fd2a 2024-11-19T11:15:30,563 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/d6190f5502c74856aca7f1ac7645fd2a, entries=150, sequenceid=358, filesize=12.0 K 2024-11-19T11:15:30,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/f3a8dde940564e38b98061861662ebc8 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/f3a8dde940564e38b98061861662ebc8 2024-11-19T11:15:30,569 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/f3a8dde940564e38b98061861662ebc8, entries=150, sequenceid=358, filesize=12.0 K 2024-11-19T11:15:30,572 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=87.22 KB/89310 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 470ms, sequenceid=358, compaction requested=false 2024-11-19T11:15:30,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:30,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:30,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-19T11:15:30,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-19T11:15:30,577 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-19T11:15:30,578 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3920 sec 2024-11-19T11:15:30,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T11:15:30,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:30,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:30,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:30,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:30,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:30,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:30,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:30,581 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.4010 sec 2024-11-19T11:15:30,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/4e8a47a5959c495abb7abcada9cc33eb is 50, key is test_row_0/A:col10/1732014930556/Put/seqid=0 2024-11-19T11:15:30,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742130_1306 (size=14741) 2024-11-19T11:15:30,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/4e8a47a5959c495abb7abcada9cc33eb 2024-11-19T11:15:30,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/98ef2c70d5c148c48a07b9dd3bc30058 is 50, key is test_row_0/B:col10/1732014930556/Put/seqid=0 2024-11-19T11:15:30,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742131_1307 (size=12301) 2024-11-19T11:15:30,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/98ef2c70d5c148c48a07b9dd3bc30058 2024-11-19T11:15:30,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/4b9a67565ca84ceeafa0713367abe8ea is 50, key is test_row_0/C:col10/1732014930556/Put/seqid=0 2024-11-19T11:15:30,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742132_1308 (size=12301) 2024-11-19T11:15:30,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:30,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014990649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:30,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:30,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:30,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:30,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:30,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014990758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:30,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:30,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:30,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:30,923 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/9cf6cbc551c3489b95a323b29ac308ec as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/9cf6cbc551c3489b95a323b29ac308ec 2024-11-19T11:15:30,927 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/C of ae1beb1ad4dbb1059f68b61e2fd94bf8 into 9cf6cbc551c3489b95a323b29ac308ec(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:30,927 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:30,927 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/C, priority=12, startTime=1732014930050; duration=0sec 2024-11-19T11:15:30,927 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:30,927 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:C 2024-11-19T11:15:30,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:30,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014990937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:30,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:30,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014990937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:30,941 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_2' in ae1beb1ad4dbb1059f68b61e2fd94bf8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:15:30,942 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_0' in ae1beb1ad4dbb1059f68b61e2fd94bf8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:15:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:30,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014990947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:30,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014990948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:30,951 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_2' in ae1beb1ad4dbb1059f68b61e2fd94bf8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:15:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:30,951 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_1' in ae1beb1ad4dbb1059f68b61e2fd94bf8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:15:30,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:30,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:30,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:30,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014990968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:30,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:30,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:30,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:31,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/4b9a67565ca84ceeafa0713367abe8ea 2024-11-19T11:15:31,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/4e8a47a5959c495abb7abcada9cc33eb as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/4e8a47a5959c495abb7abcada9cc33eb 2024-11-19T11:15:31,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/4e8a47a5959c495abb7abcada9cc33eb, entries=200, sequenceid=377, filesize=14.4 K 2024-11-19T11:15:31,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/98ef2c70d5c148c48a07b9dd3bc30058 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/98ef2c70d5c148c48a07b9dd3bc30058 2024-11-19T11:15:31,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/98ef2c70d5c148c48a07b9dd3bc30058, entries=150, sequenceid=377, filesize=12.0 K 2024-11-19T11:15:31,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/4b9a67565ca84ceeafa0713367abe8ea as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/4b9a67565ca84ceeafa0713367abe8ea 2024-11-19T11:15:31,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/4b9a67565ca84ceeafa0713367abe8ea, entries=150, sequenceid=377, filesize=12.0 K 2024-11-19T11:15:31,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 481ms, sequenceid=377, compaction requested=true 2024-11-19T11:15:31,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:31,060 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:31,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:15:31,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:31,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:15:31,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:31,060 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:31,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:15:31,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:31,061 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:31,061 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:31,061 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/A is initiating minor compaction (all files) 2024-11-19T11:15:31,061 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/B is initiating minor compaction (all files) 2024-11-19T11:15:31,061 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/A in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:31,061 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/B in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:31,061 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/c9613d59cc854ffd8161ca83c0098ea1, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/1807f855f3144bc88a2a4b990480a182, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/4e8a47a5959c495abb7abcada9cc33eb] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=39.1 K 2024-11-19T11:15:31,061 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/f88bbef4b0d74e1187d2a8cf87d75217, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/d6190f5502c74856aca7f1ac7645fd2a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/98ef2c70d5c148c48a07b9dd3bc30058] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=36.7 K 2024-11-19T11:15:31,061 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9613d59cc854ffd8161ca83c0098ea1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732014928325 2024-11-19T11:15:31,061 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting f88bbef4b0d74e1187d2a8cf87d75217, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732014928325 2024-11-19T11:15:31,062 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting d6190f5502c74856aca7f1ac7645fd2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732014928780 2024-11-19T11:15:31,062 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1807f855f3144bc88a2a4b990480a182, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732014928780 2024-11-19T11:15:31,063 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 98ef2c70d5c148c48a07b9dd3bc30058, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732014930550 2024-11-19T11:15:31,063 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e8a47a5959c495abb7abcada9cc33eb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732014930550 2024-11-19T11:15:31,070 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#A#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:31,071 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/e416d465e1f64731b47a58934b101330 is 50, key is test_row_0/A:col10/1732014930556/Put/seqid=0 2024-11-19T11:15:31,074 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#B#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:31,074 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/b44efeee84644d8b9d8ed31dca13e6ca is 50, key is test_row_0/B:col10/1732014930556/Put/seqid=0 2024-11-19T11:15:31,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742133_1309 (size=13119) 2024-11-19T11:15:31,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742134_1310 (size=13119) 2024-11-19T11:15:31,087 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/b44efeee84644d8b9d8ed31dca13e6ca as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/b44efeee84644d8b9d8ed31dca13e6ca 2024-11-19T11:15:31,092 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/B of ae1beb1ad4dbb1059f68b61e2fd94bf8 into b44efeee84644d8b9d8ed31dca13e6ca(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:31,092 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:31,092 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/B, priority=13, startTime=1732014931060; duration=0sec 2024-11-19T11:15:31,092 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:31,092 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:B 2024-11-19T11:15:31,092 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:31,093 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:31,093 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/C is initiating minor compaction (all files) 2024-11-19T11:15:31,093 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/C in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:31,093 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/9cf6cbc551c3489b95a323b29ac308ec, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/f3a8dde940564e38b98061861662ebc8, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/4b9a67565ca84ceeafa0713367abe8ea] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=36.7 K 2024-11-19T11:15:31,094 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cf6cbc551c3489b95a323b29ac308ec, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732014928325 2024-11-19T11:15:31,094 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting f3a8dde940564e38b98061861662ebc8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732014928780 2024-11-19T11:15:31,095 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b9a67565ca84ceeafa0713367abe8ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732014930550 2024-11-19T11:15:31,106 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#C#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:31,107 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/46bd59ef1696458db7b0ad0b0cffe3ec is 50, key is test_row_0/C:col10/1732014930556/Put/seqid=0 2024-11-19T11:15:31,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742135_1311 (size=13119) 2024-11-19T11:15:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:31,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T11:15:31,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:31,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:31,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:31,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:31,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:31,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:31,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/cbd54369a2bd4afab2caa8d5db53c07e is 50, key is test_row_0/A:col10/1732014931280/Put/seqid=0 2024-11-19T11:15:31,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742136_1312 (size=12301) 2024-11-19T11:15:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T11:15:31,332 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:15:31,332 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:15:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-19T11:15:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T11:15:31,334 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:15:31,335 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:15:31,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:15:31,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014991352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:31,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:31,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:31,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T11:15:31,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014991467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:31,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:31,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:31,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:31,482 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/e416d465e1f64731b47a58934b101330 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/e416d465e1f64731b47a58934b101330 2024-11-19T11:15:31,486 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/A of ae1beb1ad4dbb1059f68b61e2fd94bf8 into e416d465e1f64731b47a58934b101330(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:31,486 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:31,486 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/A, priority=13, startTime=1732014931060; duration=0sec 2024-11-19T11:15:31,486 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:31,486 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:A 2024-11-19T11:15:31,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T11:15:31,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:31,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:31,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:31,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:31,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:31,518 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/46bd59ef1696458db7b0ad0b0cffe3ec as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/46bd59ef1696458db7b0ad0b0cffe3ec 2024-11-19T11:15:31,522 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/C of ae1beb1ad4dbb1059f68b61e2fd94bf8 into 46bd59ef1696458db7b0ad0b0cffe3ec(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:31,522 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:31,522 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/C, priority=13, startTime=1732014931060; duration=0sec 2024-11-19T11:15:31,522 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:31,522 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:C 2024-11-19T11:15:31,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T11:15:31,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:31,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:31,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:31,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:31,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T11:15:31,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014991677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:31,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:31,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:31,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:31,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/cbd54369a2bd4afab2caa8d5db53c07e 2024-11-19T11:15:31,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/327b3aa6197942219feeb8bfe78e967c is 50, key is test_row_0/B:col10/1732014931280/Put/seqid=0 2024-11-19T11:15:31,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742137_1313 (size=12301) 2024-11-19T11:15:31,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/327b3aa6197942219feeb8bfe78e967c 2024-11-19T11:15:31,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/6a45fcf447704996aeb47c26447ff77e is 50, key is test_row_0/C:col10/1732014931280/Put/seqid=0 2024-11-19T11:15:31,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742138_1314 (size=12301) 2024-11-19T11:15:31,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T11:15:31,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:31,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:31,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:31,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:31,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:31,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:31,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T11:15:31,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:31,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:31,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:31,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:31,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:31,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:31,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T11:15:31,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:31,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014991987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:32,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T11:15:32,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:32,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:32,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:32,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:32,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:32,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:32,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/6a45fcf447704996aeb47c26447ff77e 2024-11-19T11:15:32,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/cbd54369a2bd4afab2caa8d5db53c07e as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/cbd54369a2bd4afab2caa8d5db53c07e 2024-11-19T11:15:32,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/cbd54369a2bd4afab2caa8d5db53c07e, entries=150, sequenceid=395, filesize=12.0 K 2024-11-19T11:15:32,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/327b3aa6197942219feeb8bfe78e967c as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/327b3aa6197942219feeb8bfe78e967c 2024-11-19T11:15:32,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/327b3aa6197942219feeb8bfe78e967c, entries=150, sequenceid=395, filesize=12.0 K 2024-11-19T11:15:32,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/6a45fcf447704996aeb47c26447ff77e as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/6a45fcf447704996aeb47c26447ff77e 2024-11-19T11:15:32,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/6a45fcf447704996aeb47c26447ff77e, entries=150, sequenceid=395, filesize=12.0 K 2024-11-19T11:15:32,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 874ms, sequenceid=395, compaction requested=false 2024-11-19T11:15:32,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:32,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T11:15:32,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:32,252 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T11:15:32,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:32,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:32,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:32,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:32,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:32,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:32,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/d0d6f8de162b4daf876267487c7757dd is 50, key is test_row_0/A:col10/1732014931339/Put/seqid=0 2024-11-19T11:15:32,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742139_1315 (size=12301) 2024-11-19T11:15:32,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T11:15:32,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:32,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:32,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:32,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014992568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:32,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:32,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:32,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:32,662 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/d0d6f8de162b4daf876267487c7757dd 2024-11-19T11:15:32,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/1f3402771c90413aba7e2eb3a8d4f082 is 50, key is test_row_0/B:col10/1732014931339/Put/seqid=0 2024-11-19T11:15:32,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:32,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014992678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:32,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:32,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:32,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:32,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742140_1316 (size=12301) 2024-11-19T11:15:32,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:32,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014992888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:32,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:32,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:32,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:32,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:32,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014992947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:32,951 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_2' in ae1beb1ad4dbb1059f68b61e2fd94bf8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:15:32,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:32,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:32,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:32,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014992967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:32,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014992968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:32,970 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_0' in ae1beb1ad4dbb1059f68b61e2fd94bf8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:15:32,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:32,970 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_1' in ae1beb1ad4dbb1059f68b61e2fd94bf8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:15:32,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:32,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014992977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:32,980 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_2' in ae1beb1ad4dbb1059f68b61e2fd94bf8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:15:32,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:32,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:32,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:33,092 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/1f3402771c90413aba7e2eb3a8d4f082 2024-11-19T11:15:33,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/cff0a19dbbca4576a99eb51e8e54942b is 50, key is test_row_0/C:col10/1732014931339/Put/seqid=0 2024-11-19T11:15:33,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742141_1317 (size=12301) 2024-11-19T11:15:33,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:33,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014993198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:33,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:33,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:33,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:33,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T11:15:33,523 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/cff0a19dbbca4576a99eb51e8e54942b 2024-11-19T11:15:33,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/d0d6f8de162b4daf876267487c7757dd as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/d0d6f8de162b4daf876267487c7757dd 2024-11-19T11:15:33,529 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/d0d6f8de162b4daf876267487c7757dd, entries=150, sequenceid=413, filesize=12.0 K 2024-11-19T11:15:33,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/1f3402771c90413aba7e2eb3a8d4f082 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/1f3402771c90413aba7e2eb3a8d4f082 2024-11-19T11:15:33,533 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/1f3402771c90413aba7e2eb3a8d4f082, entries=150, sequenceid=413, filesize=12.0 K 2024-11-19T11:15:33,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/cff0a19dbbca4576a99eb51e8e54942b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/cff0a19dbbca4576a99eb51e8e54942b 2024-11-19T11:15:33,538 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/cff0a19dbbca4576a99eb51e8e54942b, entries=150, sequenceid=413, filesize=12.0 K 2024-11-19T11:15:33,539 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 1287ms, sequenceid=413, compaction requested=true 2024-11-19T11:15:33,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:33,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:33,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-19T11:15:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-19T11:15:33,542 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-19T11:15:33,543 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2060 sec 2024-11-19T11:15:33,545 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.2120 sec 2024-11-19T11:15:33,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:33,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T11:15:33,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:33,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:33,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:33,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:33,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:33,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:33,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/c07a0f6755dc4116977e1b8ac567321b is 50, key is test_row_0/A:col10/1732014932538/Put/seqid=0 2024-11-19T11:15:33,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742142_1318 (size=14741) 2024-11-19T11:15:33,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:33,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014993783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:33,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:33,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:33,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:33,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014993897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:34,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:34,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014994108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:34,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:34,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:34,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/c07a0f6755dc4116977e1b8ac567321b 2024-11-19T11:15:34,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/ad80a2e6fde24b8fb08b96b2cd26306b is 50, key is test_row_0/B:col10/1732014932538/Put/seqid=0 2024-11-19T11:15:34,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742143_1319 (size=12301) 2024-11-19T11:15:34,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:34,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014994417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:34,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:34,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:34,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:34,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/ad80a2e6fde24b8fb08b96b2cd26306b 2024-11-19T11:15:34,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/16fa7b23ccc545fa8e2fe47d7adf0507 is 50, key is test_row_0/C:col10/1732014932538/Put/seqid=0 2024-11-19T11:15:34,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742144_1320 (size=12301) 2024-11-19T11:15:34,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/16fa7b23ccc545fa8e2fe47d7adf0507 2024-11-19T11:15:34,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/c07a0f6755dc4116977e1b8ac567321b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/c07a0f6755dc4116977e1b8ac567321b 2024-11-19T11:15:34,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/c07a0f6755dc4116977e1b8ac567321b, entries=200, sequenceid=430, filesize=14.4 K 2024-11-19T11:15:34,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/ad80a2e6fde24b8fb08b96b2cd26306b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/ad80a2e6fde24b8fb08b96b2cd26306b 2024-11-19T11:15:34,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/ad80a2e6fde24b8fb08b96b2cd26306b, entries=150, sequenceid=430, filesize=12.0 K 2024-11-19T11:15:34,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/16fa7b23ccc545fa8e2fe47d7adf0507 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/16fa7b23ccc545fa8e2fe47d7adf0507 2024-11-19T11:15:34,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/16fa7b23ccc545fa8e2fe47d7adf0507, entries=150, sequenceid=430, filesize=12.0 K 2024-11-19T11:15:34,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 932ms, sequenceid=430, compaction requested=true 2024-11-19T11:15:34,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:34,645 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:15:34,646 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52462 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:15:34,646 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/A is initiating minor compaction (all files) 2024-11-19T11:15:34,646 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/A in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:34,646 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/e416d465e1f64731b47a58934b101330, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/cbd54369a2bd4afab2caa8d5db53c07e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/d0d6f8de162b4daf876267487c7757dd, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/c07a0f6755dc4116977e1b8ac567321b] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=51.2 K 2024-11-19T11:15:34,647 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting e416d465e1f64731b47a58934b101330, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732014930550 2024-11-19T11:15:34,647 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbd54369a2bd4afab2caa8d5db53c07e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732014930642 2024-11-19T11:15:34,647 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0d6f8de162b4daf876267487c7757dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732014931327 2024-11-19T11:15:34,648 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting c07a0f6755dc4116977e1b8ac567321b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732014932538 2024-11-19T11:15:34,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:15:34,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:34,648 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:15:34,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:15:34,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:34,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:15:34,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:34,651 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:15:34,651 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/B is initiating minor compaction (all files) 2024-11-19T11:15:34,651 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/B in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:34,651 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/b44efeee84644d8b9d8ed31dca13e6ca, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/327b3aa6197942219feeb8bfe78e967c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/1f3402771c90413aba7e2eb3a8d4f082, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/ad80a2e6fde24b8fb08b96b2cd26306b] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=48.8 K 2024-11-19T11:15:34,652 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting b44efeee84644d8b9d8ed31dca13e6ca, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732014930550 2024-11-19T11:15:34,654 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 327b3aa6197942219feeb8bfe78e967c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732014930642 2024-11-19T11:15:34,654 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f3402771c90413aba7e2eb3a8d4f082, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732014931327 2024-11-19T11:15:34,655 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting ad80a2e6fde24b8fb08b96b2cd26306b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732014932538 2024-11-19T11:15:34,675 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#A#compaction#278 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:34,676 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/8a3b7bceb692430494e60033df6ef8d4 is 50, key is test_row_0/A:col10/1732014932538/Put/seqid=0 2024-11-19T11:15:34,679 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#B#compaction#279 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:34,680 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/680914e4c97147a0a1b66db421a5657a is 50, key is test_row_0/B:col10/1732014932538/Put/seqid=0 2024-11-19T11:15:34,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742145_1321 (size=13255) 2024-11-19T11:15:34,715 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/8a3b7bceb692430494e60033df6ef8d4 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/8a3b7bceb692430494e60033df6ef8d4 2024-11-19T11:15:34,723 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/A of ae1beb1ad4dbb1059f68b61e2fd94bf8 into 8a3b7bceb692430494e60033df6ef8d4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:34,723 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:34,723 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/A, priority=12, startTime=1732014934645; duration=0sec 2024-11-19T11:15:34,723 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:34,723 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:A 2024-11-19T11:15:34,723 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:15:34,724 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:15:34,725 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/C is initiating minor compaction (all files) 2024-11-19T11:15:34,725 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/C in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:34,725 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/46bd59ef1696458db7b0ad0b0cffe3ec, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/6a45fcf447704996aeb47c26447ff77e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/cff0a19dbbca4576a99eb51e8e54942b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/16fa7b23ccc545fa8e2fe47d7adf0507] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=48.8 K 2024-11-19T11:15:34,725 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46bd59ef1696458db7b0ad0b0cffe3ec, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732014930550 2024-11-19T11:15:34,725 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a45fcf447704996aeb47c26447ff77e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732014930642 2024-11-19T11:15:34,726 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting cff0a19dbbca4576a99eb51e8e54942b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732014931327 2024-11-19T11:15:34,726 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16fa7b23ccc545fa8e2fe47d7adf0507, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732014932538 2024-11-19T11:15:34,744 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#C#compaction#280 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:34,745 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/ede3e4aa10704d7e9c90ade64d46dd51 is 50, key is test_row_0/C:col10/1732014932538/Put/seqid=0 2024-11-19T11:15:34,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742146_1322 (size=13255) 2024-11-19T11:15:34,753 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/680914e4c97147a0a1b66db421a5657a as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/680914e4c97147a0a1b66db421a5657a 2024-11-19T11:15:34,763 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/B of ae1beb1ad4dbb1059f68b61e2fd94bf8 into 680914e4c97147a0a1b66db421a5657a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:34,763 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:34,763 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/B, priority=12, startTime=1732014934648; duration=0sec 2024-11-19T11:15:34,764 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:34,764 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:B 2024-11-19T11:15:34,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742147_1323 (size=13255) 2024-11-19T11:15:34,809 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/ede3e4aa10704d7e9c90ade64d46dd51 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/ede3e4aa10704d7e9c90ade64d46dd51 2024-11-19T11:15:34,816 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/C of ae1beb1ad4dbb1059f68b61e2fd94bf8 into ede3e4aa10704d7e9c90ade64d46dd51(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:34,816 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:34,816 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/C, priority=12, startTime=1732014934648; duration=0sec 2024-11-19T11:15:34,816 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:34,816 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:C 2024-11-19T11:15:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:34,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T11:15:34,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:34,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:34,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:34,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:34,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:34,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:34,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/bb55d69ebb8c439898bef2a73b03a9b5 is 50, key is test_row_0/A:col10/1732014933768/Put/seqid=0 2024-11-19T11:15:34,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742148_1324 (size=14741) 2024-11-19T11:15:34,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014994995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:34,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:34,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:34,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:35,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014995108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:35,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014995317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:35,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/bb55d69ebb8c439898bef2a73b03a9b5 2024-11-19T11:15:35,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/da71938af99b44dc98ff7d340c4de271 is 50, key is test_row_0/B:col10/1732014933768/Put/seqid=0 2024-11-19T11:15:35,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742149_1325 (size=12301) 2024-11-19T11:15:35,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/da71938af99b44dc98ff7d340c4de271 2024-11-19T11:15:35,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/5d4bff51c8c249f68839816a774034f9 is 50, key is test_row_0/C:col10/1732014933768/Put/seqid=0 2024-11-19T11:15:35,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742150_1326 (size=12301) 2024-11-19T11:15:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T11:15:35,488 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:15:35,488 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:15:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-19T11:15:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T11:15:35,490 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:15:35,490 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:15:35,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:15:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T11:15:35,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014995627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:35,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:35,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T11:15:35,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:35,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:35,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:35,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:35,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:35,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/5d4bff51c8c249f68839816a774034f9 2024-11-19T11:15:35,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/bb55d69ebb8c439898bef2a73b03a9b5 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/bb55d69ebb8c439898bef2a73b03a9b5 2024-11-19T11:15:35,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/bb55d69ebb8c439898bef2a73b03a9b5, entries=200, sequenceid=449, filesize=14.4 K 2024-11-19T11:15:35,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/da71938af99b44dc98ff7d340c4de271 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/da71938af99b44dc98ff7d340c4de271 2024-11-19T11:15:35,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T11:15:35,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:35,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:35,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:35,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:35,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:35,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/da71938af99b44dc98ff7d340c4de271, entries=150, sequenceid=449, filesize=12.0 K 2024-11-19T11:15:35,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/5d4bff51c8c249f68839816a774034f9 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/5d4bff51c8c249f68839816a774034f9 2024-11-19T11:15:35,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/5d4bff51c8c249f68839816a774034f9, entries=150, sequenceid=449, filesize=12.0 K 2024-11-19T11:15:35,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 867ms, sequenceid=449, compaction requested=false 2024-11-19T11:15:35,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T11:15:35,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T11:15:35,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:35,948 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T11:15:35,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:35,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:35,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:35,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:35,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:35,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:35,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/fb82e5c3835a470fb9b0c38c0128e131 is 50, key is test_row_0/A:col10/1732014934983/Put/seqid=0 2024-11-19T11:15:35,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742151_1327 (size=12301) 2024-11-19T11:15:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T11:15:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:36,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. as already flushing 2024-11-19T11:15:36,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014996191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:36,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:36,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:36,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:36,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:36,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014996297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:36,357 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/fb82e5c3835a470fb9b0c38c0128e131 2024-11-19T11:15:36,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/19c360c5bc5d4b318c5021048a8f4129 is 50, key is test_row_0/B:col10/1732014934983/Put/seqid=0 2024-11-19T11:15:36,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742152_1328 (size=12301) 2024-11-19T11:15:36,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014996507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T11:15:36,774 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/19c360c5bc5d4b318c5021048a8f4129 2024-11-19T11:15:36,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/5f462509898b46efbddb57326ef4b29b is 50, key is test_row_0/C:col10/1732014934983/Put/seqid=0 2024-11-19T11:15:36,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742153_1329 (size=12301) 2024-11-19T11:15:36,790 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/5f462509898b46efbddb57326ef4b29b 2024-11-19T11:15:36,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/fb82e5c3835a470fb9b0c38c0128e131 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/fb82e5c3835a470fb9b0c38c0128e131 2024-11-19T11:15:36,800 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/fb82e5c3835a470fb9b0c38c0128e131, entries=150, sequenceid=465, filesize=12.0 K 2024-11-19T11:15:36,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/19c360c5bc5d4b318c5021048a8f4129 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/19c360c5bc5d4b318c5021048a8f4129 2024-11-19T11:15:36,807 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/19c360c5bc5d4b318c5021048a8f4129, entries=150, sequenceid=465, filesize=12.0 K 2024-11-19T11:15:36,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/5f462509898b46efbddb57326ef4b29b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/5f462509898b46efbddb57326ef4b29b 2024-11-19T11:15:36,812 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/5f462509898b46efbddb57326ef4b29b, entries=150, sequenceid=465, filesize=12.0 K 2024-11-19T11:15:36,812 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 864ms, sequenceid=465, compaction requested=true 2024-11-19T11:15:36,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:36,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:36,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-19T11:15:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-19T11:15:36,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-19T11:15:36,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3240 sec 2024-11-19T11:15:36,819 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.3300 sec 2024-11-19T11:15:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:36,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T11:15:36,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:36,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:36,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:36,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:36,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:36,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:36,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/7838f1036cd24bd596aae2b95a7a8b22 is 50, key is test_row_0/A:col10/1732014936189/Put/seqid=0 2024-11-19T11:15:36,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742154_1330 (size=14741) 2024-11-19T11:15:36,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:36,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014996887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:36,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:36,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:36,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:36,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:36,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56742 deadline: 1732014996967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:36,972 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_2' in ae1beb1ad4dbb1059f68b61e2fd94bf8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:15:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:36,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:36,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:36,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:36,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56776 deadline: 1732014996987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:36,992 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_0' in ae1beb1ad4dbb1059f68b61e2fd94bf8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:15:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:37,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:37,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014996998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:37,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:37,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1732014997009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:37,013 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_1' in ae1beb1ad4dbb1059f68b61e2fd94bf8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:15:37,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:37,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:37,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:37,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:37,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732014997027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:37,031 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_2' in ae1beb1ad4dbb1059f68b61e2fd94bf8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:15:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:37,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014997207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:37,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:37,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/7838f1036cd24bd596aae2b95a7a8b22 2024-11-19T11:15:37,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/90de04dde54d416fabcb666dfccb0ebd is 50, key is test_row_0/B:col10/1732014936189/Put/seqid=0 2024-11-19T11:15:37,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742155_1331 (size=12301) 2024-11-19T11:15:37,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/90de04dde54d416fabcb666dfccb0ebd 2024-11-19T11:15:37,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/613ce771fb964fa7a92f54eeb88ae16b is 50, key is test_row_0/C:col10/1732014936189/Put/seqid=0 2024-11-19T11:15:37,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742156_1332 (size=12301) 2024-11-19T11:15:37,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56780 deadline: 1732014997518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:37,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:37,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:37,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:15:37,524 INFO [Thread-1115 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1115. 2024-11-19T11:15:37,524 DEBUG [Thread-1115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:15:37,524 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:37,524 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:37,524 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:15:37,525 INFO [Thread-1121 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1121. 2024-11-19T11:15:37,525 DEBUG [Thread-1121 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:15:37,525 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:37,525 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:37,525 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:15:37,526 INFO [Thread-1117 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1117. 2024-11-19T11:15:37,526 DEBUG [Thread-1117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:15:37,526 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:37,526 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:37,526 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:15:37,528 INFO [Thread-1113 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1113. 2024-11-19T11:15:37,528 DEBUG [Thread-1113 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:15:37,528 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:37,528 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:37,528 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:15:37,529 INFO [Thread-1119 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1119. 2024-11-19T11:15:37,529 DEBUG [Thread-1119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:15:37,529 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:37,529 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:37,529 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:15:37,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T11:15:37,637 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:15:37,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/613ce771fb964fa7a92f54eeb88ae16b 2024-11-19T11:15:37,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/7838f1036cd24bd596aae2b95a7a8b22 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/7838f1036cd24bd596aae2b95a7a8b22 2024-11-19T11:15:37,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/7838f1036cd24bd596aae2b95a7a8b22, entries=200, sequenceid=482, filesize=14.4 K 2024-11-19T11:15:37,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/90de04dde54d416fabcb666dfccb0ebd as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/90de04dde54d416fabcb666dfccb0ebd 2024-11-19T11:15:37,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/90de04dde54d416fabcb666dfccb0ebd, entries=150, sequenceid=482, filesize=12.0 K 2024-11-19T11:15:37,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/613ce771fb964fa7a92f54eeb88ae16b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/613ce771fb964fa7a92f54eeb88ae16b 2024-11-19T11:15:37,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/613ce771fb964fa7a92f54eeb88ae16b, entries=150, sequenceid=482, filesize=12.0 K 2024-11-19T11:15:37,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 849ms, sequenceid=482, compaction requested=true 2024-11-19T11:15:37,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:37,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:15:37,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:37,669 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:15:37,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:15:37,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:37,669 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:15:37,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae1beb1ad4dbb1059f68b61e2fd94bf8:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:15:37,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:37,670 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:15:37,670 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55038 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:15:37,671 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/A is initiating minor compaction (all files) 2024-11-19T11:15:37,671 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/B is initiating minor compaction (all files) 2024-11-19T11:15:37,671 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/B in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:37,671 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/A in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:37,671 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/8a3b7bceb692430494e60033df6ef8d4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/bb55d69ebb8c439898bef2a73b03a9b5, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/fb82e5c3835a470fb9b0c38c0128e131, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/7838f1036cd24bd596aae2b95a7a8b22] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=53.7 K 2024-11-19T11:15:37,671 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/680914e4c97147a0a1b66db421a5657a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/da71938af99b44dc98ff7d340c4de271, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/19c360c5bc5d4b318c5021048a8f4129, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/90de04dde54d416fabcb666dfccb0ebd] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=49.0 K 2024-11-19T11:15:37,671 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a3b7bceb692430494e60033df6ef8d4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732014932538 2024-11-19T11:15:37,671 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 680914e4c97147a0a1b66db421a5657a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732014932538 2024-11-19T11:15:37,671 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting bb55d69ebb8c439898bef2a73b03a9b5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732014933768 2024-11-19T11:15:37,671 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting da71938af99b44dc98ff7d340c4de271, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732014933768 2024-11-19T11:15:37,672 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting fb82e5c3835a470fb9b0c38c0128e131, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732014934983 2024-11-19T11:15:37,672 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19c360c5bc5d4b318c5021048a8f4129, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732014934983 2024-11-19T11:15:37,672 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 7838f1036cd24bd596aae2b95a7a8b22, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1732014936178 2024-11-19T11:15:37,672 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90de04dde54d416fabcb666dfccb0ebd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1732014936184 2024-11-19T11:15:37,682 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#A#compaction#290 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:37,682 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/6c0ba523a5aa4850991fb627118ccc67 is 50, key is test_row_0/A:col10/1732014936189/Put/seqid=0 2024-11-19T11:15:37,683 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#B#compaction#291 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:37,684 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/db5cba56efb742cdba88b9a529993788 is 50, key is test_row_0/B:col10/1732014936189/Put/seqid=0 2024-11-19T11:15:37,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742157_1333 (size=13391) 2024-11-19T11:15:37,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742158_1334 (size=13391) 2024-11-19T11:15:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:38,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T11:15:38,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:38,032 INFO [Thread-1108 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1108. 2024-11-19T11:15:38,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:38,032 DEBUG [Thread-1108 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:15:38,032 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:38,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:38,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:38,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:38,032 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:38,032 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:15:38,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:38,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/95fced0379ad4848961116d12b38abbf is 50, key is test_row_0/A:col10/1732014936872/Put/seqid=0 2024-11-19T11:15:38,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742159_1335 (size=12301) 2024-11-19T11:15:38,092 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/6c0ba523a5aa4850991fb627118ccc67 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/6c0ba523a5aa4850991fb627118ccc67 2024-11-19T11:15:38,099 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/db5cba56efb742cdba88b9a529993788 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/db5cba56efb742cdba88b9a529993788 2024-11-19T11:15:38,099 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/A of ae1beb1ad4dbb1059f68b61e2fd94bf8 into 6c0ba523a5aa4850991fb627118ccc67(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:38,099 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:38,099 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/A, priority=12, startTime=1732014937669; duration=0sec 2024-11-19T11:15:38,100 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:38,100 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:A 2024-11-19T11:15:38,100 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:15:38,101 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:15:38,101 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): ae1beb1ad4dbb1059f68b61e2fd94bf8/C is initiating minor compaction (all files) 2024-11-19T11:15:38,101 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae1beb1ad4dbb1059f68b61e2fd94bf8/C in TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:38,102 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/ede3e4aa10704d7e9c90ade64d46dd51, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/5d4bff51c8c249f68839816a774034f9, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/5f462509898b46efbddb57326ef4b29b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/613ce771fb964fa7a92f54eeb88ae16b] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp, totalSize=49.0 K 2024-11-19T11:15:38,102 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting ede3e4aa10704d7e9c90ade64d46dd51, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732014932538 2024-11-19T11:15:38,103 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d4bff51c8c249f68839816a774034f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732014933768 2024-11-19T11:15:38,103 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f462509898b46efbddb57326ef4b29b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732014934983 2024-11-19T11:15:38,104 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 613ce771fb964fa7a92f54eeb88ae16b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1732014936184 2024-11-19T11:15:38,105 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/B of ae1beb1ad4dbb1059f68b61e2fd94bf8 into db5cba56efb742cdba88b9a529993788(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:38,105 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:38,105 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/B, priority=12, startTime=1732014937669; duration=0sec 2024-11-19T11:15:38,105 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:38,105 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:B 2024-11-19T11:15:38,111 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae1beb1ad4dbb1059f68b61e2fd94bf8#C#compaction#293 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:38,112 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/7af93ae0785743ebb0482a0a10d29ada is 50, key is test_row_0/C:col10/1732014936189/Put/seqid=0 2024-11-19T11:15:38,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742160_1336 (size=13391) 2024-11-19T11:15:38,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/95fced0379ad4848961116d12b38abbf 2024-11-19T11:15:38,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/54cbc7a2a42f407eacc6c6649b8fcd35 is 50, key is test_row_0/B:col10/1732014936872/Put/seqid=0 2024-11-19T11:15:38,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742161_1337 (size=12301) 2024-11-19T11:15:38,519 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/7af93ae0785743ebb0482a0a10d29ada as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/7af93ae0785743ebb0482a0a10d29ada 2024-11-19T11:15:38,525 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae1beb1ad4dbb1059f68b61e2fd94bf8/C of ae1beb1ad4dbb1059f68b61e2fd94bf8 into 7af93ae0785743ebb0482a0a10d29ada(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:38,525 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:38,525 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., storeName=ae1beb1ad4dbb1059f68b61e2fd94bf8/C, priority=12, startTime=1732014937669; duration=0sec 2024-11-19T11:15:38,525 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:38,525 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae1beb1ad4dbb1059f68b61e2fd94bf8:C 2024-11-19T11:15:38,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/54cbc7a2a42f407eacc6c6649b8fcd35 2024-11-19T11:15:38,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/571316bf62874d14a5e7ba974eb30c69 is 50, key is test_row_0/C:col10/1732014936872/Put/seqid=0 2024-11-19T11:15:38,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742162_1338 (size=12301) 2024-11-19T11:15:39,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/571316bf62874d14a5e7ba974eb30c69 2024-11-19T11:15:39,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/95fced0379ad4848961116d12b38abbf as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/95fced0379ad4848961116d12b38abbf 2024-11-19T11:15:39,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/95fced0379ad4848961116d12b38abbf, entries=150, sequenceid=498, filesize=12.0 K 2024-11-19T11:15:39,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/54cbc7a2a42f407eacc6c6649b8fcd35 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/54cbc7a2a42f407eacc6c6649b8fcd35 2024-11-19T11:15:39,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/54cbc7a2a42f407eacc6c6649b8fcd35, entries=150, sequenceid=498, filesize=12.0 K 2024-11-19T11:15:39,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/571316bf62874d14a5e7ba974eb30c69 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/571316bf62874d14a5e7ba974eb30c69 2024-11-19T11:15:39,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/571316bf62874d14a5e7ba974eb30c69, entries=150, sequenceid=498, filesize=12.0 K 2024-11-19T11:15:39,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 1262ms, sequenceid=498, compaction requested=false 2024-11-19T11:15:39,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: 2024-11-19T11:15:43,566 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T11:15:47,028 INFO [Thread-1102 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1102. 2024-11-19T11:15:47,029 DEBUG [Thread-1102 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:15:47,029 DEBUG [Thread-1102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:47,029 DEBUG [Thread-1102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:47,029 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:15:47,079 INFO [Thread-1106 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1106. 2024-11-19T11:15:47,079 DEBUG [Thread-1106 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:15:47,079 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:47,079 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:47,079 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:15:47,109 INFO [Thread-1104 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1104. 2024-11-19T11:15:47,109 DEBUG [Thread-1104 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:15:47,109 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:47,109 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:47,109 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:15:47,119 INFO [Thread-1110 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1110. 2024-11-19T11:15:47,119 DEBUG [Thread-1110 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:15:47,119 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:47,119 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:47,120 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:15:47,120 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T11:15:47,120 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-19T11:15:47,120 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-19T11:15:47,120 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-19T11:15:47,120 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 191 2024-11-19T11:15:47,120 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-19T11:15:47,120 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T11:15:47,120 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2912 2024-11-19T11:15:47,120 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2939 2024-11-19T11:15:47,120 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2951 2024-11-19T11:15:47,120 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2904 2024-11-19T11:15:47,120 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2947 2024-11-19T11:15:47,120 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T11:15:47,120 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T11:15:47,120 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T11:15:47,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:47,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:47,120 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:15:47,121 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T11:15:47,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T11:15:47,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T11:15:47,125 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014947125"}]},"ts":"1732014947125"} 2024-11-19T11:15:47,127 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T11:15:47,127 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T11:15:47,128 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T11:15:47,130 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae1beb1ad4dbb1059f68b61e2fd94bf8, UNASSIGN}] 2024-11-19T11:15:47,131 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae1beb1ad4dbb1059f68b61e2fd94bf8, UNASSIGN 2024-11-19T11:15:47,132 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=ae1beb1ad4dbb1059f68b61e2fd94bf8, regionState=CLOSING, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:15:47,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae1beb1ad4dbb1059f68b61e2fd94bf8, UNASSIGN because future has completed 2024-11-19T11:15:47,135 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T11:15:47,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640}] 2024-11-19T11:15:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T11:15:47,288 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:47,288 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T11:15:47,288 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing ae1beb1ad4dbb1059f68b61e2fd94bf8, disabling compactions & flushes 2024-11-19T11:15:47,288 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:47,288 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:47,288 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. after waiting 0 ms 2024-11-19T11:15:47,288 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:47,289 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-19T11:15:47,289 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=A 2024-11-19T11:15:47,289 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:47,289 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=B 2024-11-19T11:15:47,289 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:47,289 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae1beb1ad4dbb1059f68b61e2fd94bf8, store=C 2024-11-19T11:15:47,289 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:47,292 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/49f2bfdac77f4f8a90cf294c25036405 is 50, key is test_row_0/A:col10/1732014947028/Put/seqid=0 2024-11-19T11:15:47,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742163_1339 (size=12301) 2024-11-19T11:15:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T11:15:47,696 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/49f2bfdac77f4f8a90cf294c25036405 2024-11-19T11:15:47,702 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/af9834ec0a8a49e089d958bd02bca241 is 50, key is test_row_0/B:col10/1732014947028/Put/seqid=0 2024-11-19T11:15:47,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742164_1340 (size=12301) 2024-11-19T11:15:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T11:15:48,107 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/af9834ec0a8a49e089d958bd02bca241 2024-11-19T11:15:48,119 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/ea143f4bd0394a1ba24937d9a6e71ba6 is 50, key is test_row_0/C:col10/1732014947028/Put/seqid=0 2024-11-19T11:15:48,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742165_1341 (size=12301) 2024-11-19T11:15:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T11:15:48,523 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/ea143f4bd0394a1ba24937d9a6e71ba6 2024-11-19T11:15:48,527 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/A/49f2bfdac77f4f8a90cf294c25036405 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/49f2bfdac77f4f8a90cf294c25036405 2024-11-19T11:15:48,530 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/49f2bfdac77f4f8a90cf294c25036405, entries=150, sequenceid=508, filesize=12.0 K 2024-11-19T11:15:48,531 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/B/af9834ec0a8a49e089d958bd02bca241 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/af9834ec0a8a49e089d958bd02bca241 2024-11-19T11:15:48,534 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/af9834ec0a8a49e089d958bd02bca241, entries=150, sequenceid=508, filesize=12.0 K 2024-11-19T11:15:48,535 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/.tmp/C/ea143f4bd0394a1ba24937d9a6e71ba6 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/ea143f4bd0394a1ba24937d9a6e71ba6 2024-11-19T11:15:48,538 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/ea143f4bd0394a1ba24937d9a6e71ba6, entries=150, sequenceid=508, filesize=12.0 K 2024-11-19T11:15:48,539 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 1250ms, sequenceid=508, compaction requested=true 2024-11-19T11:15:48,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/67764469e6ef4e0ab7b3cc4cf5f3646c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/821f195e780043aca27adbb771452130, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/bd80dc53232d4f1da7efa6b43cb3ec89, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/275099fd77f44219b0c80d2f7856de3a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/c1559d5ef47a4eb3a5847f04e5d47335, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/9db7b8fa016e4c37aeb03b87e0e4d5a9, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/1f736e7129d54e258cbddf5a55533299, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/325ee4ecceef40bf93f338d9d6a887ff, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/f447dfb6161b477ab0a1934de6004a3e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/cb2b9f20e8714cffb0b88e957709d6d8, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/9efbb97ee5ad4fc58f972546a460bf43, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/0d938065d96745a2804b6ce8772d255d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/64d12c4973184096902714c2d1fad516, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/424a7d04f3b04298b45bff5226366087, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/1cc921af62c642eb9f35c8a2406fddfd, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/cc1d2393bbb046a3a0d808214a4872ac, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/4f8b0f56241840f68f920fb440717211, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/fae6b89d02a641538b684cf2b4ad0b5f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/216c4d395da847c7ac2378ebac0ba5ad, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/b9cb5ba95b3743cfbc99c5d356b969d2, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/c9613d59cc854ffd8161ca83c0098ea1, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/f70008e8e6f445849631145c6e4e538d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/1807f855f3144bc88a2a4b990480a182, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/4e8a47a5959c495abb7abcada9cc33eb, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/e416d465e1f64731b47a58934b101330, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/cbd54369a2bd4afab2caa8d5db53c07e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/d0d6f8de162b4daf876267487c7757dd, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/c07a0f6755dc4116977e1b8ac567321b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/8a3b7bceb692430494e60033df6ef8d4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/bb55d69ebb8c439898bef2a73b03a9b5, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/fb82e5c3835a470fb9b0c38c0128e131, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/7838f1036cd24bd596aae2b95a7a8b22] to archive 2024-11-19T11:15:48,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T11:15:48,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/67764469e6ef4e0ab7b3cc4cf5f3646c to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/67764469e6ef4e0ab7b3cc4cf5f3646c 2024-11-19T11:15:48,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/821f195e780043aca27adbb771452130 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/821f195e780043aca27adbb771452130 2024-11-19T11:15:48,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/bd80dc53232d4f1da7efa6b43cb3ec89 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/bd80dc53232d4f1da7efa6b43cb3ec89 2024-11-19T11:15:48,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/275099fd77f44219b0c80d2f7856de3a to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/275099fd77f44219b0c80d2f7856de3a 2024-11-19T11:15:48,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/c1559d5ef47a4eb3a5847f04e5d47335 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/c1559d5ef47a4eb3a5847f04e5d47335 2024-11-19T11:15:48,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/9db7b8fa016e4c37aeb03b87e0e4d5a9 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/9db7b8fa016e4c37aeb03b87e0e4d5a9 2024-11-19T11:15:48,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/1f736e7129d54e258cbddf5a55533299 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/1f736e7129d54e258cbddf5a55533299 2024-11-19T11:15:48,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/325ee4ecceef40bf93f338d9d6a887ff to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/325ee4ecceef40bf93f338d9d6a887ff 2024-11-19T11:15:48,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/f447dfb6161b477ab0a1934de6004a3e to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/f447dfb6161b477ab0a1934de6004a3e 2024-11-19T11:15:48,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/cb2b9f20e8714cffb0b88e957709d6d8 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/cb2b9f20e8714cffb0b88e957709d6d8 2024-11-19T11:15:48,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/9efbb97ee5ad4fc58f972546a460bf43 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/9efbb97ee5ad4fc58f972546a460bf43 2024-11-19T11:15:48,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/0d938065d96745a2804b6ce8772d255d to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/0d938065d96745a2804b6ce8772d255d 2024-11-19T11:15:48,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/64d12c4973184096902714c2d1fad516 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/64d12c4973184096902714c2d1fad516 2024-11-19T11:15:48,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/424a7d04f3b04298b45bff5226366087 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/424a7d04f3b04298b45bff5226366087 2024-11-19T11:15:48,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/1cc921af62c642eb9f35c8a2406fddfd to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/1cc921af62c642eb9f35c8a2406fddfd 2024-11-19T11:15:48,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/cc1d2393bbb046a3a0d808214a4872ac to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/cc1d2393bbb046a3a0d808214a4872ac 2024-11-19T11:15:48,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/4f8b0f56241840f68f920fb440717211 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/4f8b0f56241840f68f920fb440717211 2024-11-19T11:15:48,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/fae6b89d02a641538b684cf2b4ad0b5f to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/fae6b89d02a641538b684cf2b4ad0b5f 2024-11-19T11:15:48,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/216c4d395da847c7ac2378ebac0ba5ad to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/216c4d395da847c7ac2378ebac0ba5ad 2024-11-19T11:15:48,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/b9cb5ba95b3743cfbc99c5d356b969d2 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/b9cb5ba95b3743cfbc99c5d356b969d2 2024-11-19T11:15:48,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/c9613d59cc854ffd8161ca83c0098ea1 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/c9613d59cc854ffd8161ca83c0098ea1 2024-11-19T11:15:48,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/f70008e8e6f445849631145c6e4e538d to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/f70008e8e6f445849631145c6e4e538d 2024-11-19T11:15:48,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/1807f855f3144bc88a2a4b990480a182 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/1807f855f3144bc88a2a4b990480a182 2024-11-19T11:15:48,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/4e8a47a5959c495abb7abcada9cc33eb to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/4e8a47a5959c495abb7abcada9cc33eb 2024-11-19T11:15:48,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/e416d465e1f64731b47a58934b101330 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/e416d465e1f64731b47a58934b101330 2024-11-19T11:15:48,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/cbd54369a2bd4afab2caa8d5db53c07e to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/cbd54369a2bd4afab2caa8d5db53c07e 2024-11-19T11:15:48,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/d0d6f8de162b4daf876267487c7757dd to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/d0d6f8de162b4daf876267487c7757dd 2024-11-19T11:15:48,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/c07a0f6755dc4116977e1b8ac567321b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/c07a0f6755dc4116977e1b8ac567321b 2024-11-19T11:15:48,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/8a3b7bceb692430494e60033df6ef8d4 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/8a3b7bceb692430494e60033df6ef8d4 2024-11-19T11:15:48,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/bb55d69ebb8c439898bef2a73b03a9b5 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/bb55d69ebb8c439898bef2a73b03a9b5 2024-11-19T11:15:48,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/fb82e5c3835a470fb9b0c38c0128e131 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/fb82e5c3835a470fb9b0c38c0128e131 2024-11-19T11:15:48,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/7838f1036cd24bd596aae2b95a7a8b22 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/7838f1036cd24bd596aae2b95a7a8b22 2024-11-19T11:15:48,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/93bfcde5b1674b3fa3424be3d09db528, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/ca7209e01dd248cfb272cf26dacef384, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/fcbdd7ae788c4920ad8556e0d1cee4d4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/a9f906f41c8e4d3fb3ecfa0c08c6b81a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/a4e3a7102e82424a90346eec0da165cc, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/f625d0d012894534b558f8c0387f3c24, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/2dad97dd207f495b99cba9204a4c66d0, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/128e2a3e0f294d8fa9d43606c38dbdab, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/cd224fc0e4dc4ac5997890f73cef932f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/bd626fc427864160abe94c2719beae51, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/9aad18f7297148ca8c9a101ae57cd767, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/2ff70e6af620454da4c721aa87fac722, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/fbae115eef9f488d805a6136c1855bd4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/e48c7e82ffac4a0b8c9cfbaec3221dd2, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/2809b4cd197c4950b2cbc46bbefe0666, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/c622c2f2c95c4b8bb28a83974645f0fa, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/2673b9e8a2664a15b70fb48ac8873492, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/ce62572c1a324c94a11b5fbf76ada02d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/abb798a2a19043daae7ab1afec337a61, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/e81c1f0622e445e2ba3c2f4c11a8c51c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/f88bbef4b0d74e1187d2a8cf87d75217, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/0dd5e2106fca4e60808aa478a43466c0, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/d6190f5502c74856aca7f1ac7645fd2a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/b44efeee84644d8b9d8ed31dca13e6ca, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/98ef2c70d5c148c48a07b9dd3bc30058, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/327b3aa6197942219feeb8bfe78e967c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/1f3402771c90413aba7e2eb3a8d4f082, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/680914e4c97147a0a1b66db421a5657a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/ad80a2e6fde24b8fb08b96b2cd26306b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/da71938af99b44dc98ff7d340c4de271, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/19c360c5bc5d4b318c5021048a8f4129, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/90de04dde54d416fabcb666dfccb0ebd] to archive 2024-11-19T11:15:48,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T11:15:48,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/93bfcde5b1674b3fa3424be3d09db528 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/93bfcde5b1674b3fa3424be3d09db528 2024-11-19T11:15:48,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/ca7209e01dd248cfb272cf26dacef384 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/ca7209e01dd248cfb272cf26dacef384 2024-11-19T11:15:48,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/fcbdd7ae788c4920ad8556e0d1cee4d4 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/fcbdd7ae788c4920ad8556e0d1cee4d4 2024-11-19T11:15:48,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/a9f906f41c8e4d3fb3ecfa0c08c6b81a to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/a9f906f41c8e4d3fb3ecfa0c08c6b81a 2024-11-19T11:15:48,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/a4e3a7102e82424a90346eec0da165cc to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/a4e3a7102e82424a90346eec0da165cc 2024-11-19T11:15:48,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/f625d0d012894534b558f8c0387f3c24 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/f625d0d012894534b558f8c0387f3c24 2024-11-19T11:15:48,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/2dad97dd207f495b99cba9204a4c66d0 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/2dad97dd207f495b99cba9204a4c66d0 2024-11-19T11:15:48,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/128e2a3e0f294d8fa9d43606c38dbdab to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/128e2a3e0f294d8fa9d43606c38dbdab 2024-11-19T11:15:48,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/cd224fc0e4dc4ac5997890f73cef932f to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/cd224fc0e4dc4ac5997890f73cef932f 2024-11-19T11:15:48,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/bd626fc427864160abe94c2719beae51 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/bd626fc427864160abe94c2719beae51 2024-11-19T11:15:48,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/9aad18f7297148ca8c9a101ae57cd767 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/9aad18f7297148ca8c9a101ae57cd767 2024-11-19T11:15:48,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/2ff70e6af620454da4c721aa87fac722 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/2ff70e6af620454da4c721aa87fac722 2024-11-19T11:15:48,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/fbae115eef9f488d805a6136c1855bd4 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/fbae115eef9f488d805a6136c1855bd4 2024-11-19T11:15:48,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/e48c7e82ffac4a0b8c9cfbaec3221dd2 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/e48c7e82ffac4a0b8c9cfbaec3221dd2 2024-11-19T11:15:48,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/2809b4cd197c4950b2cbc46bbefe0666 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/2809b4cd197c4950b2cbc46bbefe0666 2024-11-19T11:15:48,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/c622c2f2c95c4b8bb28a83974645f0fa to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/c622c2f2c95c4b8bb28a83974645f0fa 2024-11-19T11:15:48,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/2673b9e8a2664a15b70fb48ac8873492 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/2673b9e8a2664a15b70fb48ac8873492 2024-11-19T11:15:48,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/ce62572c1a324c94a11b5fbf76ada02d to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/ce62572c1a324c94a11b5fbf76ada02d 2024-11-19T11:15:48,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/abb798a2a19043daae7ab1afec337a61 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/abb798a2a19043daae7ab1afec337a61 2024-11-19T11:15:48,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/e81c1f0622e445e2ba3c2f4c11a8c51c to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/e81c1f0622e445e2ba3c2f4c11a8c51c 2024-11-19T11:15:48,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/f88bbef4b0d74e1187d2a8cf87d75217 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/f88bbef4b0d74e1187d2a8cf87d75217 2024-11-19T11:15:48,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/0dd5e2106fca4e60808aa478a43466c0 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/0dd5e2106fca4e60808aa478a43466c0 2024-11-19T11:15:48,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/d6190f5502c74856aca7f1ac7645fd2a to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/d6190f5502c74856aca7f1ac7645fd2a 2024-11-19T11:15:48,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/b44efeee84644d8b9d8ed31dca13e6ca to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/b44efeee84644d8b9d8ed31dca13e6ca 2024-11-19T11:15:48,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/98ef2c70d5c148c48a07b9dd3bc30058 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/98ef2c70d5c148c48a07b9dd3bc30058 2024-11-19T11:15:48,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/327b3aa6197942219feeb8bfe78e967c to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/327b3aa6197942219feeb8bfe78e967c 2024-11-19T11:15:48,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/1f3402771c90413aba7e2eb3a8d4f082 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/1f3402771c90413aba7e2eb3a8d4f082 2024-11-19T11:15:48,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/680914e4c97147a0a1b66db421a5657a to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/680914e4c97147a0a1b66db421a5657a 2024-11-19T11:15:48,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/ad80a2e6fde24b8fb08b96b2cd26306b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/ad80a2e6fde24b8fb08b96b2cd26306b 2024-11-19T11:15:48,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/da71938af99b44dc98ff7d340c4de271 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/da71938af99b44dc98ff7d340c4de271 2024-11-19T11:15:48,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/19c360c5bc5d4b318c5021048a8f4129 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/19c360c5bc5d4b318c5021048a8f4129 2024-11-19T11:15:48,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/90de04dde54d416fabcb666dfccb0ebd to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/90de04dde54d416fabcb666dfccb0ebd 2024-11-19T11:15:48,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/e387d356496c4f4d9f1383fc7d15b97d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/c4c177daa1cd4c15b1a64c1a2222d6fd, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/bbf7d7b4d29e46088952e1a0bbffe7c3, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/14704031a463467ab7c8f6879af211eb, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/a079e8b87fac46c687f279caf1994ddc, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/0d3c64898aff40f2ba4fa1eefbdce613, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/b6ec4078f5194a1fb222cdad1ba6f415, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/ffb6c0abdde9409bb25e2f1064ea31a5, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/d47362c958fa4ddb93e71b9bfe0f88f8, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/819ec6fca0874da4ad2c4322580c00d8, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/b5003d8669954ad888af670e22f9b41d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/4c910a28d756430faa4d824a937521a2, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/91734aab1f404fb2af836c067b54e6c7, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/5bcee6e80d294763bfaa6d05b0aaef14, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/ec1fcef53b16414b81fbc36dc46b3630, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/a1f35b9ab82648e9bd792f6c1570c6ca, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/6e80057f18fb46599418627ee47094e0, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/902b3ed4792d4ab2abce526b5847bd83, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/50981b1937fb49d3a748fbe44c5f493a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/e7b6c65861ac4cf8ae324229170788bc, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/9cf6cbc551c3489b95a323b29ac308ec, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/d43f8475b4464af6925556617b9894ed, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/f3a8dde940564e38b98061861662ebc8, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/46bd59ef1696458db7b0ad0b0cffe3ec, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/4b9a67565ca84ceeafa0713367abe8ea, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/6a45fcf447704996aeb47c26447ff77e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/cff0a19dbbca4576a99eb51e8e54942b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/ede3e4aa10704d7e9c90ade64d46dd51, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/16fa7b23ccc545fa8e2fe47d7adf0507, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/5d4bff51c8c249f68839816a774034f9, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/5f462509898b46efbddb57326ef4b29b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/613ce771fb964fa7a92f54eeb88ae16b] to archive 2024-11-19T11:15:48,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T11:15:48,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/e387d356496c4f4d9f1383fc7d15b97d to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/e387d356496c4f4d9f1383fc7d15b97d 2024-11-19T11:15:48,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/c4c177daa1cd4c15b1a64c1a2222d6fd to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/c4c177daa1cd4c15b1a64c1a2222d6fd 2024-11-19T11:15:48,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/bbf7d7b4d29e46088952e1a0bbffe7c3 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/bbf7d7b4d29e46088952e1a0bbffe7c3 2024-11-19T11:15:48,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/14704031a463467ab7c8f6879af211eb to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/14704031a463467ab7c8f6879af211eb 2024-11-19T11:15:48,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/a079e8b87fac46c687f279caf1994ddc to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/a079e8b87fac46c687f279caf1994ddc 2024-11-19T11:15:48,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/0d3c64898aff40f2ba4fa1eefbdce613 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/0d3c64898aff40f2ba4fa1eefbdce613 2024-11-19T11:15:48,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/b6ec4078f5194a1fb222cdad1ba6f415 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/b6ec4078f5194a1fb222cdad1ba6f415 2024-11-19T11:15:48,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/ffb6c0abdde9409bb25e2f1064ea31a5 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/ffb6c0abdde9409bb25e2f1064ea31a5 2024-11-19T11:15:48,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/d47362c958fa4ddb93e71b9bfe0f88f8 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/d47362c958fa4ddb93e71b9bfe0f88f8 2024-11-19T11:15:48,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/819ec6fca0874da4ad2c4322580c00d8 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/819ec6fca0874da4ad2c4322580c00d8 2024-11-19T11:15:48,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/b5003d8669954ad888af670e22f9b41d to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/b5003d8669954ad888af670e22f9b41d 2024-11-19T11:15:48,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/4c910a28d756430faa4d824a937521a2 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/4c910a28d756430faa4d824a937521a2 2024-11-19T11:15:48,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/91734aab1f404fb2af836c067b54e6c7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/91734aab1f404fb2af836c067b54e6c7 2024-11-19T11:15:48,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/5bcee6e80d294763bfaa6d05b0aaef14 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/5bcee6e80d294763bfaa6d05b0aaef14 2024-11-19T11:15:48,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/ec1fcef53b16414b81fbc36dc46b3630 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/ec1fcef53b16414b81fbc36dc46b3630 2024-11-19T11:15:48,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/a1f35b9ab82648e9bd792f6c1570c6ca to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/a1f35b9ab82648e9bd792f6c1570c6ca 2024-11-19T11:15:48,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/6e80057f18fb46599418627ee47094e0 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/6e80057f18fb46599418627ee47094e0 2024-11-19T11:15:48,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/902b3ed4792d4ab2abce526b5847bd83 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/902b3ed4792d4ab2abce526b5847bd83 2024-11-19T11:15:48,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/50981b1937fb49d3a748fbe44c5f493a to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/50981b1937fb49d3a748fbe44c5f493a 2024-11-19T11:15:48,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/e7b6c65861ac4cf8ae324229170788bc to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/e7b6c65861ac4cf8ae324229170788bc 2024-11-19T11:15:48,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/9cf6cbc551c3489b95a323b29ac308ec to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/9cf6cbc551c3489b95a323b29ac308ec 2024-11-19T11:15:48,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/d43f8475b4464af6925556617b9894ed to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/d43f8475b4464af6925556617b9894ed 2024-11-19T11:15:48,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/f3a8dde940564e38b98061861662ebc8 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/f3a8dde940564e38b98061861662ebc8 2024-11-19T11:15:48,635 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/46bd59ef1696458db7b0ad0b0cffe3ec to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/46bd59ef1696458db7b0ad0b0cffe3ec 2024-11-19T11:15:48,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/4b9a67565ca84ceeafa0713367abe8ea to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/4b9a67565ca84ceeafa0713367abe8ea 2024-11-19T11:15:48,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/6a45fcf447704996aeb47c26447ff77e to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/6a45fcf447704996aeb47c26447ff77e 2024-11-19T11:15:48,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/cff0a19dbbca4576a99eb51e8e54942b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/cff0a19dbbca4576a99eb51e8e54942b 2024-11-19T11:15:48,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/ede3e4aa10704d7e9c90ade64d46dd51 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/ede3e4aa10704d7e9c90ade64d46dd51 2024-11-19T11:15:48,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/16fa7b23ccc545fa8e2fe47d7adf0507 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/16fa7b23ccc545fa8e2fe47d7adf0507 2024-11-19T11:15:48,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/5d4bff51c8c249f68839816a774034f9 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/5d4bff51c8c249f68839816a774034f9 2024-11-19T11:15:48,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/5f462509898b46efbddb57326ef4b29b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/5f462509898b46efbddb57326ef4b29b 2024-11-19T11:15:48,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/613ce771fb964fa7a92f54eeb88ae16b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/613ce771fb964fa7a92f54eeb88ae16b 2024-11-19T11:15:48,650 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/recovered.edits/511.seqid, newMaxSeqId=511, maxSeqId=1 2024-11-19T11:15:48,650 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. 2024-11-19T11:15:48,650 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for ae1beb1ad4dbb1059f68b61e2fd94bf8: Waiting for close lock at 1732014947288Running coprocessor pre-close hooks at 1732014947288Disabling compacts and flushes for region at 1732014947288Disabling writes for close at 1732014947288Obtaining lock to block concurrent updates at 1732014947289 (+1 ms)Preparing flush snapshotting stores in ae1beb1ad4dbb1059f68b61e2fd94bf8 at 1732014947289Finished memstore snapshotting TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732014947289Flushing stores of TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8. at 1732014947289Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8/A: creating writer at 1732014947289Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8/A: appending metadata at 1732014947292 (+3 ms)Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8/A: closing flushed file at 1732014947292Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8/B: creating writer at 1732014947700 (+408 ms)Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8/B: appending metadata at 1732014947702 (+2 ms)Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8/B: closing flushed file at 1732014947702Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8/C: creating writer at 1732014948114 (+412 ms)Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8/C: appending metadata at 1732014948118 (+4 ms)Flushing ae1beb1ad4dbb1059f68b61e2fd94bf8/C: closing flushed file at 1732014948118Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40ee8670: reopening flushed file at 1732014948526 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b763baa: reopening flushed file at 1732014948530 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48f7d053: reopening flushed file at 1732014948534 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ae1beb1ad4dbb1059f68b61e2fd94bf8 in 1250ms, sequenceid=508, compaction requested=true at 1732014948539 (+5 ms)Writing region close event to WAL at 1732014948646 (+107 ms)Running coprocessor post-close hooks at 1732014948650 (+4 ms)Closed at 1732014948650 2024-11-19T11:15:48,652 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:48,653 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=ae1beb1ad4dbb1059f68b61e2fd94bf8, regionState=CLOSED 2024-11-19T11:15:48,654 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 because future has completed 2024-11-19T11:15:48,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-19T11:15:48,657 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure ae1beb1ad4dbb1059f68b61e2fd94bf8, server=16c88966c0af,35833,1732014855640 in 1.5200 sec 2024-11-19T11:15:48,658 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-19T11:15:48,658 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae1beb1ad4dbb1059f68b61e2fd94bf8, UNASSIGN in 1.5270 sec 2024-11-19T11:15:48,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-19T11:15:48,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5310 sec 2024-11-19T11:15:48,661 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014948660"}]},"ts":"1732014948660"} 2024-11-19T11:15:48,662 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T11:15:48,662 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T11:15:48,663 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5410 sec 2024-11-19T11:15:49,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T11:15:49,269 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:15:49,270 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T11:15:49,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:15:49,275 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:15:49,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T11:15:49,276 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:15:49,279 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:49,282 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A, FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B, FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C, FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/recovered.edits] 2024-11-19T11:15:49,285 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/49f2bfdac77f4f8a90cf294c25036405 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/49f2bfdac77f4f8a90cf294c25036405 2024-11-19T11:15:49,286 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/6c0ba523a5aa4850991fb627118ccc67 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/6c0ba523a5aa4850991fb627118ccc67 2024-11-19T11:15:49,287 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/95fced0379ad4848961116d12b38abbf to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/A/95fced0379ad4848961116d12b38abbf 2024-11-19T11:15:49,290 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/54cbc7a2a42f407eacc6c6649b8fcd35 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/54cbc7a2a42f407eacc6c6649b8fcd35 2024-11-19T11:15:49,291 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/af9834ec0a8a49e089d958bd02bca241 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/af9834ec0a8a49e089d958bd02bca241 2024-11-19T11:15:49,293 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/db5cba56efb742cdba88b9a529993788 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/B/db5cba56efb742cdba88b9a529993788 2024-11-19T11:15:49,295 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/571316bf62874d14a5e7ba974eb30c69 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/571316bf62874d14a5e7ba974eb30c69 2024-11-19T11:15:49,297 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/7af93ae0785743ebb0482a0a10d29ada to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/7af93ae0785743ebb0482a0a10d29ada 2024-11-19T11:15:49,299 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/ea143f4bd0394a1ba24937d9a6e71ba6 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/C/ea143f4bd0394a1ba24937d9a6e71ba6 2024-11-19T11:15:49,303 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/recovered.edits/511.seqid to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8/recovered.edits/511.seqid 2024-11-19T11:15:49,304 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/ae1beb1ad4dbb1059f68b61e2fd94bf8 2024-11-19T11:15:49,304 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T11:15:49,307 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:15:49,313 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T11:15:49,315 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T11:15:49,316 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:15:49,317 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T11:15:49,317 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732014949317"}]},"ts":"9223372036854775807"} 2024-11-19T11:15:49,319 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T11:15:49,319 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ae1beb1ad4dbb1059f68b61e2fd94bf8, NAME => 'TestAcidGuarantees,,1732014915331.ae1beb1ad4dbb1059f68b61e2fd94bf8.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T11:15:49,319 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T11:15:49,319 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732014949319"}]},"ts":"9223372036854775807"} 2024-11-19T11:15:49,320 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T11:15:49,321 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:15:49,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 51 msec 2024-11-19T11:15:49,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T11:15:49,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T11:15:49,388 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:15:49,399 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 252), OpenFileDescriptor=445 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=418 (was 380) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=12244 (was 11768) - AvailableMemoryMB LEAK? - 2024-11-19T11:15:49,409 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=418, ProcessCount=11, AvailableMemoryMB=12243 2024-11-19T11:15:49,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T11:15:49,411 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:15:49,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T11:15:49,413 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T11:15:49,414 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:49,414 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-19T11:15:49,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-19T11:15:49,415 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T11:15:49,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742166_1342 (size=1061) 2024-11-19T11:15:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-19T11:15:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-19T11:15:49,825 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094 2024-11-19T11:15:49,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742167_1343 (size=53) 2024-11-19T11:15:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-19T11:15:50,237 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:15:50,237 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d631ed8d9a4ea7a066117289c5e10132, disabling compactions & flushes 2024-11-19T11:15:50,237 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:50,237 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:50,237 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. after waiting 0 ms 2024-11-19T11:15:50,237 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:50,237 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:50,237 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d631ed8d9a4ea7a066117289c5e10132: Waiting for close lock at 1732014950237Disabling compacts and flushes for region at 1732014950237Disabling writes for close at 1732014950237Writing region close event to WAL at 1732014950237Closed at 1732014950237 2024-11-19T11:15:50,239 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T11:15:50,240 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732014950239"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732014950239"}]},"ts":"1732014950239"} 2024-11-19T11:15:50,242 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T11:15:50,244 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T11:15:50,244 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014950244"}]},"ts":"1732014950244"} 2024-11-19T11:15:50,247 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T11:15:50,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d631ed8d9a4ea7a066117289c5e10132, ASSIGN}] 2024-11-19T11:15:50,249 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d631ed8d9a4ea7a066117289c5e10132, ASSIGN 2024-11-19T11:15:50,250 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d631ed8d9a4ea7a066117289c5e10132, ASSIGN; state=OFFLINE, location=16c88966c0af,35833,1732014855640; forceNewPlan=false, retain=false 2024-11-19T11:15:50,401 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=d631ed8d9a4ea7a066117289c5e10132, regionState=OPENING, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:15:50,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d631ed8d9a4ea7a066117289c5e10132, ASSIGN because future has completed 2024-11-19T11:15:50,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640}] 2024-11-19T11:15:50,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-19T11:15:50,560 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:50,560 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} 2024-11-19T11:15:50,560 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:50,560 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:15:50,561 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:50,561 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:50,562 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:50,564 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:15:50,564 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d631ed8d9a4ea7a066117289c5e10132 columnFamilyName A 2024-11-19T11:15:50,564 DEBUG [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:50,565 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] regionserver.HStore(327): Store=d631ed8d9a4ea7a066117289c5e10132/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:15:50,565 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:50,566 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:15:50,566 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d631ed8d9a4ea7a066117289c5e10132 columnFamilyName B 2024-11-19T11:15:50,567 DEBUG [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:50,567 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] regionserver.HStore(327): Store=d631ed8d9a4ea7a066117289c5e10132/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:15:50,567 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:50,569 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:15:50,569 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d631ed8d9a4ea7a066117289c5e10132 columnFamilyName C 2024-11-19T11:15:50,569 DEBUG [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:50,570 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] regionserver.HStore(327): Store=d631ed8d9a4ea7a066117289c5e10132/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:15:50,570 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:50,570 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:50,571 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:50,571 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:50,572 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:50,572 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:50,572 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T11:15:50,573 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:50,575 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T11:15:50,575 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened d631ed8d9a4ea7a066117289c5e10132; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63685743, jitterRate=-0.05100847780704498}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T11:15:50,575 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:50,576 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for d631ed8d9a4ea7a066117289c5e10132: Running coprocessor pre-open hook at 1732014950561Writing region info on filesystem at 1732014950561Initializing all the Stores at 1732014950562 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014950562Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014950562Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014950562Cleaning up temporary data from old regions at 1732014950572 (+10 ms)Running coprocessor post-open hooks at 1732014950575 (+3 ms)Region opened successfully at 1732014950576 (+1 ms) 2024-11-19T11:15:50,577 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., pid=89, masterSystemTime=1732014950557 2024-11-19T11:15:50,579 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:50,579 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:50,586 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=d631ed8d9a4ea7a066117289c5e10132, regionState=OPEN, openSeqNum=2, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:15:50,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 because future has completed 2024-11-19T11:15:50,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-19T11:15:50,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 in 186 msec 2024-11-19T11:15:50,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-19T11:15:50,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d631ed8d9a4ea7a066117289c5e10132, ASSIGN in 347 msec 2024-11-19T11:15:50,598 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T11:15:50,599 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014950598"}]},"ts":"1732014950598"} 2024-11-19T11:15:50,602 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T11:15:50,604 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T11:15:50,607 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1930 sec 2024-11-19T11:15:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-19T11:15:51,558 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:15:51,560 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db9998b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:51,560 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:15:51,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:15:51,562 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:15:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:15:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:15:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d66eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:15:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:15:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:51,565 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:15:51,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@285838d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:51,566 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:15:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:15:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:15:51,570 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:15:51,571 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T11:15:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 16c88966c0af,41441,1732014854424 2024-11-19T11:15:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@b33c7a 2024-11-19T11:15:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T11:15:51,573 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T11:15:51,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T11:15:51,575 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:15:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T11:15:51,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742168_1344 (size=1097) 2024-11-19T11:15:51,988 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T11:15:51,988 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T11:15:51,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T11:15:51,993 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d631ed8d9a4ea7a066117289c5e10132, REOPEN/MOVE}] 2024-11-19T11:15:51,994 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d631ed8d9a4ea7a066117289c5e10132, REOPEN/MOVE 2024-11-19T11:15:51,995 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=d631ed8d9a4ea7a066117289c5e10132, regionState=CLOSING, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:15:51,996 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d631ed8d9a4ea7a066117289c5e10132, REOPEN/MOVE because future has completed 2024-11-19T11:15:51,996 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T11:15:51,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640}] 2024-11-19T11:15:52,149 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:52,149 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T11:15:52,149 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing d631ed8d9a4ea7a066117289c5e10132, disabling compactions & flushes 2024-11-19T11:15:52,149 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:52,149 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:52,149 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. after waiting 0 ms 2024-11-19T11:15:52,149 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:52,154 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T11:15:52,155 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:52,155 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for d631ed8d9a4ea7a066117289c5e10132: Waiting for close lock at 1732014952149Running coprocessor pre-close hooks at 1732014952149Disabling compacts and flushes for region at 1732014952149Disabling writes for close at 1732014952149Writing region close event to WAL at 1732014952149Running coprocessor post-close hooks at 1732014952155 (+6 ms)Closed at 1732014952155 2024-11-19T11:15:52,155 WARN [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: d631ed8d9a4ea7a066117289c5e10132 to self. 2024-11-19T11:15:52,159 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:52,159 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=d631ed8d9a4ea7a066117289c5e10132, regionState=CLOSED 2024-11-19T11:15:52,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 because future has completed 2024-11-19T11:15:52,165 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-19T11:15:52,165 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 in 167 msec 2024-11-19T11:15:52,166 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d631ed8d9a4ea7a066117289c5e10132, REOPEN/MOVE; state=CLOSED, location=16c88966c0af,35833,1732014855640; forceNewPlan=false, retain=true 2024-11-19T11:15:52,317 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=d631ed8d9a4ea7a066117289c5e10132, regionState=OPENING, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:15:52,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d631ed8d9a4ea7a066117289c5e10132, REOPEN/MOVE because future has completed 2024-11-19T11:15:52,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640}] 2024-11-19T11:15:52,473 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:52,473 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} 2024-11-19T11:15:52,474 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:52,474 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:15:52,474 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:52,474 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:52,475 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:52,476 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:15:52,476 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d631ed8d9a4ea7a066117289c5e10132 columnFamilyName A 2024-11-19T11:15:52,477 DEBUG [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:52,477 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] regionserver.HStore(327): Store=d631ed8d9a4ea7a066117289c5e10132/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:15:52,478 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:52,478 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:15:52,478 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d631ed8d9a4ea7a066117289c5e10132 columnFamilyName B 2024-11-19T11:15:52,478 DEBUG [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:52,479 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] regionserver.HStore(327): Store=d631ed8d9a4ea7a066117289c5e10132/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:15:52,479 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:52,479 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:15:52,479 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d631ed8d9a4ea7a066117289c5e10132 columnFamilyName C 2024-11-19T11:15:52,479 DEBUG [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:52,480 INFO [StoreOpener-d631ed8d9a4ea7a066117289c5e10132-1 {}] regionserver.HStore(327): Store=d631ed8d9a4ea7a066117289c5e10132/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:15:52,480 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:52,480 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:52,481 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:52,481 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:52,482 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:52,482 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:52,483 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T11:15:52,484 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:52,484 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened d631ed8d9a4ea7a066117289c5e10132; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65088110, jitterRate=-0.030111581087112427}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T11:15:52,484 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:52,485 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for d631ed8d9a4ea7a066117289c5e10132: Running coprocessor pre-open hook at 1732014952474Writing region info on filesystem at 1732014952474Initializing all the Stores at 1732014952474Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014952475 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014952475Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014952475Cleaning up temporary data from old regions at 1732014952482 (+7 ms)Running coprocessor post-open hooks at 1732014952484 (+2 ms)Region opened successfully at 1732014952485 (+1 ms) 2024-11-19T11:15:52,486 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., pid=94, masterSystemTime=1732014952471 2024-11-19T11:15:52,487 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:52,487 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:52,488 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=d631ed8d9a4ea7a066117289c5e10132, regionState=OPEN, openSeqNum=5, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:15:52,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 because future has completed 2024-11-19T11:15:52,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-19T11:15:52,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 in 171 msec 2024-11-19T11:15:52,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-19T11:15:52,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d631ed8d9a4ea7a066117289c5e10132, REOPEN/MOVE in 498 msec 2024-11-19T11:15:52,495 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-19T11:15:52,495 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 502 msec 2024-11-19T11:15:52,498 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 921 msec 2024-11-19T11:15:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-19T11:15:52,498 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:15:52,499 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0f1e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,499 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:15:52,499 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:15:52,500 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:15:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:15:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:15:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@342af2ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:15:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:15:52,501 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:52,501 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:15:52,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14d90cff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,502 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:15:52,503 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:15:52,503 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:15:52,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:15:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:15:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40641caa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:15:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:15:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:52,505 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:15:52,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@479d031, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,506 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:15:52,506 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:15:52,507 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:15:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:15:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:15:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36c7fc9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:15:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:15:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:52,508 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:15:52,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c7b04e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,509 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:15:52,509 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:15:52,510 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:15:52,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:15:52,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:15:52,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4493baee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:15:52,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:15:52,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:52,511 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:15:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14228d7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,512 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:15:52,512 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:15:52,513 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:15:52,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:15:52,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:15:52,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31b54fc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:15:52,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:15:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:52,514 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:15:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fe389ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,515 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,515 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:15:52,516 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:15:52,516 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:15:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:15:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:15:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73de7bb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:15:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:15:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:52,518 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:15:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@159752f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,519 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,519 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:15:52,519 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:15:52,520 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:15:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:15:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:15:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1751e615, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:15:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:15:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:52,521 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:15:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ccd5cd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,522 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,522 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:15:52,522 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:15:52,523 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:15:52,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:15:52,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:15:52,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c43e775, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:15:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:15:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:52,524 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:15:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b5cf238, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,526 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,526 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:15:52,526 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:15:52,526 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:15:52,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:15:52,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:15:52,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eb7df86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:15:52,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:15:52,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:52,528 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:15:52,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4270858d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa71035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,529 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:15:52,529 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:15:52,530 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:15:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:15:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:15:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ddad383, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:15:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:15:52,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:15:52,531 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:15:52,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d831ef7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:15:52,532 DEBUG [Thread-1585 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:15:52,532 DEBUG [Thread-1583 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:15:52,533 DEBUG [Thread-1577 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:15:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:15:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:15:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:15:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:15:52,533 DEBUG [Thread-1568 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:15:52,534 DEBUG [Thread-1579 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:15:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:15:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:15:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:15:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:15:52,534 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:15:52,535 DEBUG [Thread-1566 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:15:52,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:15:52,535 DEBUG [Thread-1570 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:15:52,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:15:52,535 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:15:52,536 DEBUG [Thread-1572 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:15:52,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:15:52,536 DEBUG [Thread-1570 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:15:52,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:15:52,536 DEBUG [Thread-1570 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:15:52,536 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:15:52,537 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:15:52,537 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:15:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:15:52,537 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:15:52,537 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:15:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:15:52,538 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:15:52,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:15:52,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:15:52,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:15:52,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:15:52,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:15:52,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:15:52,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:15:52,546 DEBUG [Thread-1574 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:15:52,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:15:52,548 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:15:52,548 DEBUG [Thread-1581 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:15:52,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:15:52,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:15:52,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:15:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:15:52,550 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:15:52,550 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:15:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-19T11:15:52,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:15:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T11:15:52,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:15:52,554 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:15:52,555 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:15:52,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:15:52,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d631ed8d9a4ea7a066117289c5e10132 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T11:15:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:52,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=A 2024-11-19T11:15:52,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:52,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=B 2024-11-19T11:15:52,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:52,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=C 2024-11-19T11:15:52,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:52,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f7ae1a6a184b437a899a0354903a0e23_d631ed8d9a4ea7a066117289c5e10132 is 50, key is test_row_0/A:col10/1732014952560/Put/seqid=0 2024-11-19T11:15:52,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742169_1345 (size=12154) 2024-11-19T11:15:52,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:52,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015012600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015012601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:52,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:52,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015012610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:52,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732014952580, max timestamp=1732014952610Num uniques 859; 2024-11-19T11:15:52,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732014952580, max timestamp=1732014952610Num uniques 859; 2024-11-19T11:15:52,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T11:15:52,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T11:15:52,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732014952580, max timestamp=1732014952610Num uniques 859; 2024-11-19T11:15:52,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T11:15:52,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:52,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T11:15:52,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732015012666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:52,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:52,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1732015012669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:52,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:52,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T11:15:52,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:52,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:15:52,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:52,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:52,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:52,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:52,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015012717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015012717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:52,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015012717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:52,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732015012787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:52,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1732015012787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:52,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T11:15:52,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:52,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:15:52,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:52,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:52,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T11:15:52,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015012927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:52,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015012927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:52,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015012928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:52,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:52,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:53,001 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:53,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:53,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732015012997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:53,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1732015012997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:53,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:53,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:53,005 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f7ae1a6a184b437a899a0354903a0e23_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f7ae1a6a184b437a899a0354903a0e23_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:53,006 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/bc1c6e3291e44f509a3462b45e665f8b, store: [table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:15:53,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/bc1c6e3291e44f509a3462b45e665f8b is 175, key is test_row_0/A:col10/1732014952560/Put/seqid=0 2024-11-19T11:15:53,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742170_1346 (size=30955) 2024-11-19T11:15:53,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T11:15:53,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:53,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:15:53,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:53,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:53,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:53,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:53,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T11:15:53,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:53,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:15:53,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:53,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:53,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:53,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:53,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T11:15:53,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:53,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015013247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:53,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:53,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015013247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:53,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:53,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:53,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015013247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:53,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:53,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:53,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:53,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732015013307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:53,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:53,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1732015013307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:53,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:53,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:53,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T11:15:53,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:53,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:15:53,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:53,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:53,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:53,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:53,412 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/bc1c6e3291e44f509a3462b45e665f8b 2024-11-19T11:15:53,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/d8af6d65385f48ba8cf513b7fd14eed3 is 50, key is test_row_0/B:col10/1732014952560/Put/seqid=0 2024-11-19T11:15:53,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742171_1347 (size=12001) 2024-11-19T11:15:53,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T11:15:53,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:53,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:15:53,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:53,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:53,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:53,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:53,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T11:15:53,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:53,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:15:53,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:53,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:53,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:53,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T11:15:53,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015013768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:53,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015013768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:53,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015013768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T11:15:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:53,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:53,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:15:53,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:53,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:53,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:53,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732015013818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:53,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:53,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1732015013828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:53,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/d8af6d65385f48ba8cf513b7fd14eed3 2024-11-19T11:15:53,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/8af1e6b1b2684c6781b67b3ea6562986 is 50, key is test_row_0/C:col10/1732014952560/Put/seqid=0 2024-11-19T11:15:53,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742172_1348 (size=12001) 2024-11-19T11:15:53,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T11:15:53,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:53,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:15:53,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:53,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:53,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:54,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T11:15:54,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:54,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:15:54,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:54,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:54,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:54,173 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T11:15:54,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T11:15:54,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:54,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:15:54,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:54,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:54,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:54,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/8af1e6b1b2684c6781b67b3ea6562986 2024-11-19T11:15:54,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/bc1c6e3291e44f509a3462b45e665f8b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/bc1c6e3291e44f509a3462b45e665f8b 2024-11-19T11:15:54,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/bc1c6e3291e44f509a3462b45e665f8b, entries=150, sequenceid=15, filesize=30.2 K 2024-11-19T11:15:54,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/d8af6d65385f48ba8cf513b7fd14eed3 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/d8af6d65385f48ba8cf513b7fd14eed3 2024-11-19T11:15:54,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/d8af6d65385f48ba8cf513b7fd14eed3, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T11:15:54,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/8af1e6b1b2684c6781b67b3ea6562986 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8af1e6b1b2684c6781b67b3ea6562986 2024-11-19T11:15:54,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8af1e6b1b2684c6781b67b3ea6562986, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T11:15:54,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d631ed8d9a4ea7a066117289c5e10132 in 1714ms, sequenceid=15, compaction requested=false 2024-11-19T11:15:54,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:15:54,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T11:15:54,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:54,389 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing d631ed8d9a4ea7a066117289c5e10132 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-19T11:15:54,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=A 2024-11-19T11:15:54,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:54,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=B 2024-11-19T11:15:54,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:54,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=C 2024-11-19T11:15:54,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:54,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119857066ac9d724aeaa8f3d6c5055f2a9a_d631ed8d9a4ea7a066117289c5e10132 is 50, key is test_row_0/A:col10/1732014952662/Put/seqid=0 2024-11-19T11:15:54,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742173_1349 (size=12154) 2024-11-19T11:15:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T11:15:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:54,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:15:54,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:54,807 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119857066ac9d724aeaa8f3d6c5055f2a9a_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119857066ac9d724aeaa8f3d6c5055f2a9a_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:54,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/1216e1a3ce0f4e83adb390e3e188b4a4, store: [table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:15:54,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/1216e1a3ce0f4e83adb390e3e188b4a4 is 175, key is test_row_0/A:col10/1732014952662/Put/seqid=0 2024-11-19T11:15:54,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015014805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:54,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:54,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:54,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:54,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742174_1350 (size=30955) 2024-11-19T11:15:54,821 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/1216e1a3ce0f4e83adb390e3e188b4a4 2024-11-19T11:15:54,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015014811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:54,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015014818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:54,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:54,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:54,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:54,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/1aee25267bc544179fb950c634534964 is 50, key is test_row_0/B:col10/1732014952662/Put/seqid=0 2024-11-19T11:15:54,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742175_1351 (size=12001) 2024-11-19T11:15:54,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732015014827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:54,840 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_0' in d631ed8d9a4ea7a066117289c5e10132 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:15:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:54,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:54,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:54,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1732015014838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:54,848 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_1' in d631ed8d9a4ea7a066117289c5e10132 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:15:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:54,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015014928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:54,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:54,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015014928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:54,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:54,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:54,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:54,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:54,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:54,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015014937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:54,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:54,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:54,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:55,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:55,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015015137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:55,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:55,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015015137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:55,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015015157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:55,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:55,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:55,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:55,237 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/1aee25267bc544179fb950c634534964 2024-11-19T11:15:55,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/082e9053e3884e129c23de59689ef0a2 is 50, key is test_row_0/C:col10/1732014952662/Put/seqid=0 2024-11-19T11:15:55,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742176_1352 (size=12001) 2024-11-19T11:15:55,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015015447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:55,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015015447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:55,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015015477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:55,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:55,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:55,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:55,649 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/082e9053e3884e129c23de59689ef0a2 2024-11-19T11:15:55,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/1216e1a3ce0f4e83adb390e3e188b4a4 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/1216e1a3ce0f4e83adb390e3e188b4a4 2024-11-19T11:15:55,655 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/1216e1a3ce0f4e83adb390e3e188b4a4, entries=150, sequenceid=49, filesize=30.2 K 2024-11-19T11:15:55,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/1aee25267bc544179fb950c634534964 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/1aee25267bc544179fb950c634534964 2024-11-19T11:15:55,659 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/1aee25267bc544179fb950c634534964, entries=150, sequenceid=49, filesize=11.7 K 2024-11-19T11:15:55,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/082e9053e3884e129c23de59689ef0a2 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/082e9053e3884e129c23de59689ef0a2 2024-11-19T11:15:55,662 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/082e9053e3884e129c23de59689ef0a2, entries=150, sequenceid=49, filesize=11.7 K 2024-11-19T11:15:55,662 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for d631ed8d9a4ea7a066117289c5e10132 in 1273ms, sequenceid=49, compaction requested=false 2024-11-19T11:15:55,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:15:55,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:55,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-19T11:15:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-19T11:15:55,665 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-19T11:15:55,665 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1080 sec 2024-11-19T11:15:55,666 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 3.1180 sec 2024-11-19T11:15:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:55,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d631ed8d9a4ea7a066117289c5e10132 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T11:15:55,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=A 2024-11-19T11:15:55,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:55,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=B 2024-11-19T11:15:55,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:55,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=C 2024-11-19T11:15:55,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:55,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c24ee4bb97bf4aa197d1dd94b83e9cc6_d631ed8d9a4ea7a066117289c5e10132 is 50, key is test_row_0/A:col10/1732014955972/Put/seqid=0 2024-11-19T11:15:55,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742177_1353 (size=17034) 2024-11-19T11:15:56,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015016015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:56,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:56,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015016021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:56,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:56,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:56,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015016023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:56,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:56,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015016138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015016138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015016138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:56,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015016347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:56,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015016347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:56,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015016347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:56,387 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:56,389 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c24ee4bb97bf4aa197d1dd94b83e9cc6_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c24ee4bb97bf4aa197d1dd94b83e9cc6_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:56,390 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/f59794ad2d2b4c2ba1c6bc5d908abb69, store: [table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:15:56,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/f59794ad2d2b4c2ba1c6bc5d908abb69 is 175, key is test_row_0/A:col10/1732014955972/Put/seqid=0 2024-11-19T11:15:56,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742178_1354 (size=48139) 2024-11-19T11:15:56,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015016667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:56,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015016667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015016667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T11:15:56,728 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:15:56,728 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:15:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-19T11:15:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T11:15:56,730 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:15:56,730 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:15:56,731 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:15:56,798 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/f59794ad2d2b4c2ba1c6bc5d908abb69 2024-11-19T11:15:56,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/045bfce002a546bfb189825be44ed65a is 50, key is test_row_0/B:col10/1732014955972/Put/seqid=0 2024-11-19T11:15:56,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742179_1355 (size=12001) 2024-11-19T11:15:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T11:15:56,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732015016847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:56,854 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_0' in d631ed8d9a4ea7a066117289c5e10132 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:15:56,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:56,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:56,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1732015016857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:56,870 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_1' in d631ed8d9a4ea7a066117289c5e10132 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:15:56,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:56,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:56,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T11:15:56,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:56,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:15:56,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:56,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:56,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:56,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:57,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T11:15:57,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:57,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:15:57,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:57,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:57,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T11:15:57,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015017178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:57,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:57,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:57,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:57,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T11:15:57,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:57,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:15:57,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:57,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:57,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:57,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:57,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015017188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015017188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:57,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/045bfce002a546bfb189825be44ed65a 2024-11-19T11:15:57,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/894c5d8ff650451785c7b24089cbe598 is 50, key is test_row_0/C:col10/1732014955972/Put/seqid=0 2024-11-19T11:15:57,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742180_1356 (size=12001) 2024-11-19T11:15:57,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/894c5d8ff650451785c7b24089cbe598 2024-11-19T11:15:57,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/f59794ad2d2b4c2ba1c6bc5d908abb69 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/f59794ad2d2b4c2ba1c6bc5d908abb69 2024-11-19T11:15:57,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/f59794ad2d2b4c2ba1c6bc5d908abb69, entries=250, sequenceid=62, filesize=47.0 K 2024-11-19T11:15:57,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/045bfce002a546bfb189825be44ed65a as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/045bfce002a546bfb189825be44ed65a 2024-11-19T11:15:57,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/045bfce002a546bfb189825be44ed65a, entries=150, sequenceid=62, filesize=11.7 K 2024-11-19T11:15:57,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/894c5d8ff650451785c7b24089cbe598 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/894c5d8ff650451785c7b24089cbe598 2024-11-19T11:15:57,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/894c5d8ff650451785c7b24089cbe598, entries=150, sequenceid=62, filesize=11.7 K 2024-11-19T11:15:57,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d631ed8d9a4ea7a066117289c5e10132 in 1256ms, sequenceid=62, compaction requested=true 2024-11-19T11:15:57,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:15:57,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d631ed8d9a4ea7a066117289c5e10132:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:15:57,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:57,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d631ed8d9a4ea7a066117289c5e10132:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:15:57,229 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:57,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:57,229 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:57,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d631ed8d9a4ea7a066117289c5e10132:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:15:57,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:57,230 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:57,230 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:57,230 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): d631ed8d9a4ea7a066117289c5e10132/B is initiating minor compaction (all files) 2024-11-19T11:15:57,230 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): d631ed8d9a4ea7a066117289c5e10132/A is initiating minor compaction (all files) 2024-11-19T11:15:57,230 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d631ed8d9a4ea7a066117289c5e10132/A in TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:57,230 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d631ed8d9a4ea7a066117289c5e10132/B in TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:57,230 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/bc1c6e3291e44f509a3462b45e665f8b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/1216e1a3ce0f4e83adb390e3e188b4a4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/f59794ad2d2b4c2ba1c6bc5d908abb69] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp, totalSize=107.5 K 2024-11-19T11:15:57,230 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/d8af6d65385f48ba8cf513b7fd14eed3, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/1aee25267bc544179fb950c634534964, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/045bfce002a546bfb189825be44ed65a] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp, totalSize=35.2 K 2024-11-19T11:15:57,230 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:57,230 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. files: [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/bc1c6e3291e44f509a3462b45e665f8b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/1216e1a3ce0f4e83adb390e3e188b4a4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/f59794ad2d2b4c2ba1c6bc5d908abb69] 2024-11-19T11:15:57,231 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting d8af6d65385f48ba8cf513b7fd14eed3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732014952547 2024-11-19T11:15:57,231 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc1c6e3291e44f509a3462b45e665f8b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732014952547 2024-11-19T11:15:57,231 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 1aee25267bc544179fb950c634534964, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732014952662 2024-11-19T11:15:57,231 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1216e1a3ce0f4e83adb390e3e188b4a4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732014952662 2024-11-19T11:15:57,231 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 045bfce002a546bfb189825be44ed65a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732014954803 2024-11-19T11:15:57,231 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting f59794ad2d2b4c2ba1c6bc5d908abb69, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732014954803 2024-11-19T11:15:57,237 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:15:57,239 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d631ed8d9a4ea7a066117289c5e10132#B#compaction#308 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:57,239 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/bc4ea39c11ea40089b689f07c29dfc52 is 50, key is test_row_0/B:col10/1732014955972/Put/seqid=0 2024-11-19T11:15:57,240 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119ef0be3285b74499287a69ed1745abbf9_d631ed8d9a4ea7a066117289c5e10132 store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:15:57,242 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119ef0be3285b74499287a69ed1745abbf9_d631ed8d9a4ea7a066117289c5e10132, store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:15:57,242 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ef0be3285b74499287a69ed1745abbf9_d631ed8d9a4ea7a066117289c5e10132 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:15:57,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742181_1357 (size=12104) 2024-11-19T11:15:57,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742182_1358 (size=4469) 2024-11-19T11:15:57,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T11:15:57,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:57,341 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing d631ed8d9a4ea7a066117289c5e10132 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T11:15:57,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=A 2024-11-19T11:15:57,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:57,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=B 2024-11-19T11:15:57,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:57,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=C 2024-11-19T11:15:57,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:57,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111967157c75770a48a7a20d395f489fbbc2_d631ed8d9a4ea7a066117289c5e10132 is 50, key is test_row_0/A:col10/1732014956020/Put/seqid=0 2024-11-19T11:15:57,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742183_1359 (size=12154) 2024-11-19T11:15:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T11:15:57,651 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d631ed8d9a4ea7a066117289c5e10132#A#compaction#309 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:57,652 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/f98ca166e12048efb1d209bbdb071bc5 is 175, key is test_row_0/A:col10/1732014955972/Put/seqid=0 2024-11-19T11:15:57,653 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/bc4ea39c11ea40089b689f07c29dfc52 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/bc4ea39c11ea40089b689f07c29dfc52 2024-11-19T11:15:57,659 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d631ed8d9a4ea7a066117289c5e10132/B of d631ed8d9a4ea7a066117289c5e10132 into bc4ea39c11ea40089b689f07c29dfc52(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:57,659 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:15:57,659 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., storeName=d631ed8d9a4ea7a066117289c5e10132/B, priority=13, startTime=1732014957229; duration=0sec 2024-11-19T11:15:57,659 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:57,659 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d631ed8d9a4ea7a066117289c5e10132:B 2024-11-19T11:15:57,659 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:57,660 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:57,660 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): d631ed8d9a4ea7a066117289c5e10132/C is initiating minor compaction (all files) 2024-11-19T11:15:57,660 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d631ed8d9a4ea7a066117289c5e10132/C in TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:57,660 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8af1e6b1b2684c6781b67b3ea6562986, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/082e9053e3884e129c23de59689ef0a2, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/894c5d8ff650451785c7b24089cbe598] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp, totalSize=35.2 K 2024-11-19T11:15:57,661 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8af1e6b1b2684c6781b67b3ea6562986, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732014952547 2024-11-19T11:15:57,661 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 082e9053e3884e129c23de59689ef0a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732014952662 2024-11-19T11:15:57,661 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 894c5d8ff650451785c7b24089cbe598, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732014954803 2024-11-19T11:15:57,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742184_1360 (size=31058) 2024-11-19T11:15:57,667 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/f98ca166e12048efb1d209bbdb071bc5 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/f98ca166e12048efb1d209bbdb071bc5 2024-11-19T11:15:57,669 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d631ed8d9a4ea7a066117289c5e10132#C#compaction#311 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:57,670 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/8fa44d0b59744a158b6c75ed217fa706 is 50, key is test_row_0/C:col10/1732014955972/Put/seqid=0 2024-11-19T11:15:57,671 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d631ed8d9a4ea7a066117289c5e10132/A of d631ed8d9a4ea7a066117289c5e10132 into f98ca166e12048efb1d209bbdb071bc5(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:57,671 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:15:57,671 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., storeName=d631ed8d9a4ea7a066117289c5e10132/A, priority=13, startTime=1732014957229; duration=0sec 2024-11-19T11:15:57,671 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:57,671 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d631ed8d9a4ea7a066117289c5e10132:A 2024-11-19T11:15:57,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742185_1361 (size=12104) 2024-11-19T11:15:57,677 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/8fa44d0b59744a158b6c75ed217fa706 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8fa44d0b59744a158b6c75ed217fa706 2024-11-19T11:15:57,693 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d631ed8d9a4ea7a066117289c5e10132/C of d631ed8d9a4ea7a066117289c5e10132 into 8fa44d0b59744a158b6c75ed217fa706(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:57,694 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:15:57,694 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., storeName=d631ed8d9a4ea7a066117289c5e10132/C, priority=13, startTime=1732014957229; duration=0sec 2024-11-19T11:15:57,694 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:57,694 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d631ed8d9a4ea7a066117289c5e10132:C 2024-11-19T11:15:57,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:57,757 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111967157c75770a48a7a20d395f489fbbc2_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111967157c75770a48a7a20d395f489fbbc2_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:57,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/c8e18728acb44f42901d881a6180220b, store: [table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:15:57,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/c8e18728acb44f42901d881a6180220b is 175, key is test_row_0/A:col10/1732014956020/Put/seqid=0 2024-11-19T11:15:57,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742186_1362 (size=30955) 2024-11-19T11:15:57,766 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/c8e18728acb44f42901d881a6180220b 2024-11-19T11:15:57,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/185e9d03282d461c95cadad500dbeafc is 50, key is test_row_0/B:col10/1732014956020/Put/seqid=0 2024-11-19T11:15:57,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742187_1363 (size=12001) 2024-11-19T11:15:57,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T11:15:58,177 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/185e9d03282d461c95cadad500dbeafc 2024-11-19T11:15:58,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/d7cafa7074c94e128676a8d63bd7353a is 50, key is test_row_0/C:col10/1732014956020/Put/seqid=0 2024-11-19T11:15:58,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742188_1364 (size=12001) 2024-11-19T11:15:58,188 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/d7cafa7074c94e128676a8d63bd7353a 2024-11-19T11:15:58,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/c8e18728acb44f42901d881a6180220b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/c8e18728acb44f42901d881a6180220b 2024-11-19T11:15:58,205 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/c8e18728acb44f42901d881a6180220b, entries=150, sequenceid=81, filesize=30.2 K 2024-11-19T11:15:58,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/185e9d03282d461c95cadad500dbeafc as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/185e9d03282d461c95cadad500dbeafc 2024-11-19T11:15:58,211 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/185e9d03282d461c95cadad500dbeafc, entries=150, sequenceid=81, filesize=11.7 K 2024-11-19T11:15:58,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/d7cafa7074c94e128676a8d63bd7353a as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/d7cafa7074c94e128676a8d63bd7353a 2024-11-19T11:15:58,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,216 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/d7cafa7074c94e128676a8d63bd7353a, entries=150, sequenceid=81, filesize=11.7 K 2024-11-19T11:15:58,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,217 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for d631ed8d9a4ea7a066117289c5e10132 in 876ms, sequenceid=81, compaction requested=false 2024-11-19T11:15:58,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:15:58,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:58,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-19T11:15:58,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-19T11:15:58,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-19T11:15:58,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4880 sec 2024-11-19T11:15:58,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,226 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.4970 sec 2024-11-19T11:15:58,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:58,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,312 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d631ed8d9a4ea7a066117289c5e10132 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T11:15:58,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=A 2024-11-19T11:15:58,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:58,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=B 2024-11-19T11:15:58,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:58,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=C 2024-11-19T11:15:58,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:15:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bc7e028fb80e4693a4b646e93e888cd0_d631ed8d9a4ea7a066117289c5e10132 is 50, key is test_row_0/A:col10/1732014958311/Put/seqid=0 2024-11-19T11:15:58,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742189_1365 (size=21918) 2024-11-19T11:15:58,342 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:58,346 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bc7e028fb80e4693a4b646e93e888cd0_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bc7e028fb80e4693a4b646e93e888cd0_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:58,346 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/b4567bb83a7640c68bf3bf550ce32ed2, store: [table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:15:58,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/b4567bb83a7640c68bf3bf550ce32ed2 is 175, key is test_row_0/A:col10/1732014958311/Put/seqid=0 2024-11-19T11:15:58,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742190_1366 (size=65323) 2024-11-19T11:15:58,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732014958364, max timestamp=1732014958388Num uniques 859; 2024-11-19T11:15:58,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732014958364, max timestamp=1732014958388Num uniques 859; 2024-11-19T11:15:58,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T11:15:58,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T11:15:58,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732014958364, max timestamp=1732014958388Num uniques 859; 2024-11-19T11:15:58,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T11:15:58,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015018446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:58,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015018447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:58,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015018447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:58,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015018568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:58,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015018568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:58,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:58,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:58,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015018568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:58,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:58,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:58,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:58,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:58,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:58,755 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/b4567bb83a7640c68bf3bf550ce32ed2 2024-11-19T11:15:58,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/32acf037b7644aefbb921c38dc31a9e6 is 50, key is test_row_0/B:col10/1732014958311/Put/seqid=0 2024-11-19T11:15:58,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742191_1367 (size=12001) 2024-11-19T11:15:58,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:58,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015018788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:58,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:58,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015018788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:58,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015018788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T11:15:58,877 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:15:58,878 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:15:58,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-19T11:15:58,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T11:15:58,879 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:15:58,880 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:15:58,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:15:58,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T11:15:59,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T11:15:59,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:59,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:15:59,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:59,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:59,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:59,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:59,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:59,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015019097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:59,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015019097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:59,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015019097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:59,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/32acf037b7644aefbb921c38dc31a9e6 2024-11-19T11:15:59,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/7a000616336d4940833b58397d40429d is 50, key is test_row_0/C:col10/1732014958311/Put/seqid=0 2024-11-19T11:15:59,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T11:15:59,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:59,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:15:59,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:59,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:59,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:59,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742192_1368 (size=12001) 2024-11-19T11:15:59,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/7a000616336d4940833b58397d40429d 2024-11-19T11:15:59,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:15:59,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T11:15:59,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/b4567bb83a7640c68bf3bf550ce32ed2 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/b4567bb83a7640c68bf3bf550ce32ed2 2024-11-19T11:15:59,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/b4567bb83a7640c68bf3bf550ce32ed2, entries=350, sequenceid=94, filesize=63.8 K 2024-11-19T11:15:59,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/32acf037b7644aefbb921c38dc31a9e6 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/32acf037b7644aefbb921c38dc31a9e6 2024-11-19T11:15:59,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/32acf037b7644aefbb921c38dc31a9e6, entries=150, sequenceid=94, filesize=11.7 K 2024-11-19T11:15:59,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/7a000616336d4940833b58397d40429d as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/7a000616336d4940833b58397d40429d 2024-11-19T11:15:59,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/7a000616336d4940833b58397d40429d, entries=150, sequenceid=94, filesize=11.7 K 2024-11-19T11:15:59,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d631ed8d9a4ea7a066117289c5e10132 in 901ms, sequenceid=94, compaction requested=true 2024-11-19T11:15:59,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:15:59,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d631ed8d9a4ea7a066117289c5e10132:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:15:59,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:59,214 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:59,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d631ed8d9a4ea7a066117289c5e10132:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:15:59,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:15:59,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d631ed8d9a4ea7a066117289c5e10132:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:15:59,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:59,214 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:59,215 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:59,215 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127336 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:59,215 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): d631ed8d9a4ea7a066117289c5e10132/A is initiating minor compaction (all files) 2024-11-19T11:15:59,215 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): d631ed8d9a4ea7a066117289c5e10132/B is initiating minor compaction (all files) 2024-11-19T11:15:59,215 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d631ed8d9a4ea7a066117289c5e10132/B in TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:59,215 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d631ed8d9a4ea7a066117289c5e10132/A in TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:59,215 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/f98ca166e12048efb1d209bbdb071bc5, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/c8e18728acb44f42901d881a6180220b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/b4567bb83a7640c68bf3bf550ce32ed2] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp, totalSize=124.4 K 2024-11-19T11:15:59,215 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/bc4ea39c11ea40089b689f07c29dfc52, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/185e9d03282d461c95cadad500dbeafc, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/32acf037b7644aefbb921c38dc31a9e6] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp, totalSize=35.3 K 2024-11-19T11:15:59,215 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:59,215 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. files: [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/f98ca166e12048efb1d209bbdb071bc5, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/c8e18728acb44f42901d881a6180220b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/b4567bb83a7640c68bf3bf550ce32ed2] 2024-11-19T11:15:59,215 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc4ea39c11ea40089b689f07c29dfc52, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732014954803 2024-11-19T11:15:59,215 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting f98ca166e12048efb1d209bbdb071bc5, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732014954803 2024-11-19T11:15:59,215 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting c8e18728acb44f42901d881a6180220b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732014956007 2024-11-19T11:15:59,215 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 185e9d03282d461c95cadad500dbeafc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732014956007 2024-11-19T11:15:59,216 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32acf037b7644aefbb921c38dc31a9e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732014958297 2024-11-19T11:15:59,216 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting b4567bb83a7640c68bf3bf550ce32ed2, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732014958247 2024-11-19T11:15:59,220 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d631ed8d9a4ea7a066117289c5e10132#B#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:59,221 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/be49d5608f5141e1875e91d308f5f2ee is 50, key is test_row_0/B:col10/1732014958311/Put/seqid=0 2024-11-19T11:15:59,221 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:15:59,225 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119188646aece5241dda161c802d4595c6e_d631ed8d9a4ea7a066117289c5e10132 store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:15:59,227 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119188646aece5241dda161c802d4595c6e_d631ed8d9a4ea7a066117289c5e10132, store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:15:59,227 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119188646aece5241dda161c802d4595c6e_d631ed8d9a4ea7a066117289c5e10132 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:15:59,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742193_1369 (size=12207) 2024-11-19T11:15:59,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742194_1370 (size=4469) 2024-11-19T11:15:59,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T11:15:59,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:59,340 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing d631ed8d9a4ea7a066117289c5e10132 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-19T11:15:59,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=A 2024-11-19T11:15:59,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:59,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=B 2024-11-19T11:15:59,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:59,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=C 2024-11-19T11:15:59,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:15:59,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199b2a882803e74ebfa45aec81fafb4e58_d631ed8d9a4ea7a066117289c5e10132 is 50, key is test_row_0/A:col10/1732014958446/Put/seqid=0 2024-11-19T11:15:59,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742195_1371 (size=12154) 2024-11-19T11:15:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T11:15:59,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:59,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:15:59,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:59,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015019621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:59,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:59,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015019629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:59,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:59,639 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/be49d5608f5141e1875e91d308f5f2ee as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/be49d5608f5141e1875e91d308f5f2ee 2024-11-19T11:15:59,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:59,639 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d631ed8d9a4ea7a066117289c5e10132#A#compaction#318 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:59,640 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/23f42e682e144cb58a6d7e8500637630 is 175, key is test_row_0/A:col10/1732014958311/Put/seqid=0 2024-11-19T11:15:59,643 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d631ed8d9a4ea7a066117289c5e10132/B of d631ed8d9a4ea7a066117289c5e10132 into be49d5608f5141e1875e91d308f5f2ee(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:15:59,643 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:15:59,643 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., storeName=d631ed8d9a4ea7a066117289c5e10132/B, priority=13, startTime=1732014959214; duration=0sec 2024-11-19T11:15:59,643 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:15:59,643 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d631ed8d9a4ea7a066117289c5e10132:B 2024-11-19T11:15:59,643 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:15:59,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:59,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015019633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:59,644 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:15:59,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,644 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): d631ed8d9a4ea7a066117289c5e10132/C is initiating minor compaction (all files) 2024-11-19T11:15:59,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:59,645 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d631ed8d9a4ea7a066117289c5e10132/C in TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:15:59,645 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8fa44d0b59744a158b6c75ed217fa706, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/d7cafa7074c94e128676a8d63bd7353a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/7a000616336d4940833b58397d40429d] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp, totalSize=35.3 K 2024-11-19T11:15:59,645 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fa44d0b59744a158b6c75ed217fa706, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732014954803 2024-11-19T11:15:59,645 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7cafa7074c94e128676a8d63bd7353a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732014956007 2024-11-19T11:15:59,645 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a000616336d4940833b58397d40429d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732014958297 2024-11-19T11:15:59,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742196_1372 (size=31161) 2024-11-19T11:15:59,659 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d631ed8d9a4ea7a066117289c5e10132#C#compaction#320 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:15:59,660 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/599654f729614e758bfc36e2e7c3e55d is 50, key is test_row_0/C:col10/1732014958311/Put/seqid=0 2024-11-19T11:15:59,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742197_1373 (size=12207) 2024-11-19T11:15:59,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:59,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015019747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:59,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:59,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015019747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:59,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:59,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:59,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015019747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:59,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:15:59,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:59,761 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199b2a882803e74ebfa45aec81fafb4e58_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199b2a882803e74ebfa45aec81fafb4e58_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:15:59,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/9db11ed13f2f4d748d27ac4292f5bc8b, store: [table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:15:59,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/9db11ed13f2f4d748d27ac4292f5bc8b is 175, key is test_row_0/A:col10/1732014958446/Put/seqid=0 2024-11-19T11:15:59,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742198_1374 (size=30955) 2024-11-19T11:15:59,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015019967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:59,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015019967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:15:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:15:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015019967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:15:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:15:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T11:16:00,061 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/23f42e682e144cb58a6d7e8500637630 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/23f42e682e144cb58a6d7e8500637630 2024-11-19T11:16:00,064 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d631ed8d9a4ea7a066117289c5e10132/A of d631ed8d9a4ea7a066117289c5e10132 into 23f42e682e144cb58a6d7e8500637630(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:00,064 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:00,064 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., storeName=d631ed8d9a4ea7a066117289c5e10132/A, priority=13, startTime=1732014959214; duration=0sec 2024-11-19T11:16:00,064 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:00,064 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d631ed8d9a4ea7a066117289c5e10132:A 2024-11-19T11:16:00,066 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/599654f729614e758bfc36e2e7c3e55d as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/599654f729614e758bfc36e2e7c3e55d 2024-11-19T11:16:00,069 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d631ed8d9a4ea7a066117289c5e10132/C of d631ed8d9a4ea7a066117289c5e10132 into 599654f729614e758bfc36e2e7c3e55d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:00,069 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:00,069 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., storeName=d631ed8d9a4ea7a066117289c5e10132/C, priority=13, startTime=1732014959214; duration=0sec 2024-11-19T11:16:00,069 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:00,069 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d631ed8d9a4ea7a066117289c5e10132:C 2024-11-19T11:16:00,168 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/9db11ed13f2f4d748d27ac4292f5bc8b 2024-11-19T11:16:00,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/47c1aa8328774cc9b444a55b260ac65a is 50, key is test_row_0/B:col10/1732014958446/Put/seqid=0 2024-11-19T11:16:00,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742199_1375 (size=12001) 2024-11-19T11:16:00,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015020278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:00,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015020279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:00,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015020279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:00,582 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/47c1aa8328774cc9b444a55b260ac65a 2024-11-19T11:16:00,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/c30b92b28d114cd3adc3fd648a7abb77 is 50, key is test_row_0/C:col10/1732014958446/Put/seqid=0 2024-11-19T11:16:00,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742200_1376 (size=12001) 2024-11-19T11:16:00,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:00,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015020797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:00,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015020797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:00,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015020798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:00,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732015020887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:00,894 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_0' in d631ed8d9a4ea7a066117289c5e10132 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:00,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:00,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1732015020907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:00,913 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_1' in d631ed8d9a4ea7a066117289c5e10132 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:00,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:00,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:00,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:00,994 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/c30b92b28d114cd3adc3fd648a7abb77 2024-11-19T11:16:00,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/9db11ed13f2f4d748d27ac4292f5bc8b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/9db11ed13f2f4d748d27ac4292f5bc8b 2024-11-19T11:16:01,005 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/9db11ed13f2f4d748d27ac4292f5bc8b, entries=150, sequenceid=129, filesize=30.2 K 2024-11-19T11:16:01,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/47c1aa8328774cc9b444a55b260ac65a as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/47c1aa8328774cc9b444a55b260ac65a 2024-11-19T11:16:01,009 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/47c1aa8328774cc9b444a55b260ac65a, entries=150, sequenceid=129, filesize=11.7 K 2024-11-19T11:16:01,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/c30b92b28d114cd3adc3fd648a7abb77 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/c30b92b28d114cd3adc3fd648a7abb77 2024-11-19T11:16:01,017 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/c30b92b28d114cd3adc3fd648a7abb77, entries=150, sequenceid=129, filesize=11.7 K 2024-11-19T11:16:01,018 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for d631ed8d9a4ea7a066117289c5e10132 in 1678ms, sequenceid=129, compaction requested=false 2024-11-19T11:16:01,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:01,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:01,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-19T11:16:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-19T11:16:01,022 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-19T11:16:01,022 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1400 sec 2024-11-19T11:16:01,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.1440 sec 2024-11-19T11:16:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T11:16:01,028 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:16:01,028 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:16:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-19T11:16:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T11:16:01,030 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:16:01,030 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:16:01,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:16:01,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T11:16:01,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T11:16:01,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:01,182 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing d631ed8d9a4ea7a066117289c5e10132 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T11:16:01,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=A 2024-11-19T11:16:01,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:01,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=B 2024-11-19T11:16:01,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:01,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=C 2024-11-19T11:16:01,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:01,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b7e40c98d29e46a8b3571f11c861ed39_d631ed8d9a4ea7a066117289c5e10132 is 50, key is test_row_0/A:col10/1732014959631/Put/seqid=0 2024-11-19T11:16:01,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742201_1377 (size=12304) 2024-11-19T11:16:01,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T11:16:01,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:01,603 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b7e40c98d29e46a8b3571f11c861ed39_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b7e40c98d29e46a8b3571f11c861ed39_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:01,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/deeae19b57ae44cd97003123c1213d72, store: [table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:01,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/deeae19b57ae44cd97003123c1213d72 is 175, key is test_row_0/A:col10/1732014959631/Put/seqid=0 2024-11-19T11:16:01,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742202_1378 (size=31105) 2024-11-19T11:16:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T11:16:01,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:01,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:16:01,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732014961853, max timestamp=1732014961871Num uniques 859; 2024-11-19T11:16:01,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732014961853, max timestamp=1732014961871Num uniques 859; 2024-11-19T11:16:01,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732014961853, max timestamp=1732014961871Num uniques 859; 2024-11-19T11:16:01,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T11:16:01,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T11:16:01,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T11:16:01,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015021922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:01,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015021923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:01,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015021929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:02,029 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=142, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/deeae19b57ae44cd97003123c1213d72 2024-11-19T11:16:02,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/fec6bde1efe545c78afa82ce91a88b28 is 50, key is test_row_0/B:col10/1732014959631/Put/seqid=0 2024-11-19T11:16:02,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742203_1379 (size=12151) 2024-11-19T11:16:02,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015022038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:02,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015022038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:02,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:02,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015022047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:02,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T11:16:02,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:02,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015022257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015022257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:02,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:02,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015022267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:02,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:02,441 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/fec6bde1efe545c78afa82ce91a88b28 2024-11-19T11:16:02,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/f3649235dcfb4d21b0cc2f36a56b2ba7 is 50, key is test_row_0/C:col10/1732014959631/Put/seqid=0 2024-11-19T11:16:02,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742204_1380 (size=12151) 2024-11-19T11:16:02,450 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/f3649235dcfb4d21b0cc2f36a56b2ba7 2024-11-19T11:16:02,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/deeae19b57ae44cd97003123c1213d72 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/deeae19b57ae44cd97003123c1213d72 2024-11-19T11:16:02,457 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/deeae19b57ae44cd97003123c1213d72, entries=150, sequenceid=142, filesize=30.4 K 2024-11-19T11:16:02,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/fec6bde1efe545c78afa82ce91a88b28 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/fec6bde1efe545c78afa82ce91a88b28 2024-11-19T11:16:02,462 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/fec6bde1efe545c78afa82ce91a88b28, entries=150, sequenceid=142, filesize=11.9 K 2024-11-19T11:16:02,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/f3649235dcfb4d21b0cc2f36a56b2ba7 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/f3649235dcfb4d21b0cc2f36a56b2ba7 2024-11-19T11:16:02,467 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/f3649235dcfb4d21b0cc2f36a56b2ba7, entries=150, sequenceid=142, filesize=11.9 K 2024-11-19T11:16:02,468 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d631ed8d9a4ea7a066117289c5e10132 in 1286ms, sequenceid=142, compaction requested=true 2024-11-19T11:16:02,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:02,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:02,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-19T11:16:02,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-19T11:16:02,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-19T11:16:02,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4400 sec 2024-11-19T11:16:02,474 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.4440 sec 2024-11-19T11:16:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:02,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d631ed8d9a4ea7a066117289c5e10132 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-19T11:16:02,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=A 2024-11-19T11:16:02,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:02,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=B 2024-11-19T11:16:02,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:02,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=C 2024-11-19T11:16:02,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:02,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111979e3f5e7aa8c4319803072ddc732eaed_d631ed8d9a4ea7a066117289c5e10132 is 50, key is test_row_0/A:col10/1732014961921/Put/seqid=0 2024-11-19T11:16:02,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742205_1381 (size=14794) 2024-11-19T11:16:02,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:02,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015022601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:02,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015022603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:02,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:02,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015022618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:02,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:02,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015022727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:02,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:02,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015022727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:02,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:02,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015022727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:02,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:02,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:02,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015022938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:02,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015022938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015022938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:02,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:03,003 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:03,006 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111979e3f5e7aa8c4319803072ddc732eaed_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111979e3f5e7aa8c4319803072ddc732eaed_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:03,006 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/b33883e363664d4c91f48f30545be293, store: [table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:03,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/b33883e363664d4c91f48f30545be293 is 175, key is test_row_0/A:col10/1732014961921/Put/seqid=0 2024-11-19T11:16:03,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742206_1382 (size=39749) 2024-11-19T11:16:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T11:16:03,187 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:16:03,188 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:16:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-19T11:16:03,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T11:16:03,189 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:16:03,190 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:16:03,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:16:03,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015023258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:03,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:03,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:03,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015023258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:03,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:03,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:03,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:03,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:03,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015023258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:03,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:03,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:03,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T11:16:03,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T11:16:03,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:03,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:16:03,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:03,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:03,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:03,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:03,410 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=178, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/b33883e363664d4c91f48f30545be293 2024-11-19T11:16:03,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/41c5ac048b2647bf98b540e81ef29c44 is 50, key is test_row_0/B:col10/1732014961921/Put/seqid=0 2024-11-19T11:16:03,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742207_1383 (size=12151) 2024-11-19T11:16:03,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/41c5ac048b2647bf98b540e81ef29c44 2024-11-19T11:16:03,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/bf575832dd77437eb35a9edccfaff989 is 50, key is test_row_0/C:col10/1732014961921/Put/seqid=0 2024-11-19T11:16:03,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742208_1384 (size=12151) 2024-11-19T11:16:03,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T11:16:03,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:03,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:16:03,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:03,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:03,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T11:16:03,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T11:16:03,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:03,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:16:03,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:03,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:03,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:03,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:03,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015023767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:03,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:03,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:03,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:03,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015023777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:03,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015023777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:03,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T11:16:03,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:03,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:16:03,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:03,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:03,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:03,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:03,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T11:16:03,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/bf575832dd77437eb35a9edccfaff989 2024-11-19T11:16:03,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/b33883e363664d4c91f48f30545be293 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/b33883e363664d4c91f48f30545be293 2024-11-19T11:16:03,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/b33883e363664d4c91f48f30545be293, entries=200, sequenceid=178, filesize=38.8 K 2024-11-19T11:16:03,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/41c5ac048b2647bf98b540e81ef29c44 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/41c5ac048b2647bf98b540e81ef29c44 2024-11-19T11:16:03,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/41c5ac048b2647bf98b540e81ef29c44, entries=150, sequenceid=178, filesize=11.9 K 2024-11-19T11:16:03,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/bf575832dd77437eb35a9edccfaff989 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/bf575832dd77437eb35a9edccfaff989 2024-11-19T11:16:03,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/bf575832dd77437eb35a9edccfaff989, entries=150, sequenceid=178, filesize=11.9 K 2024-11-19T11:16:03,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for d631ed8d9a4ea7a066117289c5e10132 in 1323ms, sequenceid=178, compaction requested=true 2024-11-19T11:16:03,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:03,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d631ed8d9a4ea7a066117289c5e10132:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:16:03,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:03,907 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:16:03,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d631ed8d9a4ea7a066117289c5e10132:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:16:03,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:03,907 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:16:03,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d631ed8d9a4ea7a066117289c5e10132:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:16:03,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:03,909 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:16:03,909 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:16:03,909 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): d631ed8d9a4ea7a066117289c5e10132/B is initiating minor compaction (all files) 2024-11-19T11:16:03,909 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): d631ed8d9a4ea7a066117289c5e10132/A is initiating minor compaction (all files) 2024-11-19T11:16:03,909 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d631ed8d9a4ea7a066117289c5e10132/A in TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:03,909 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d631ed8d9a4ea7a066117289c5e10132/B in TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:03,909 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/23f42e682e144cb58a6d7e8500637630, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/9db11ed13f2f4d748d27ac4292f5bc8b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/deeae19b57ae44cd97003123c1213d72, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/b33883e363664d4c91f48f30545be293] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp, totalSize=129.9 K 2024-11-19T11:16:03,909 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/be49d5608f5141e1875e91d308f5f2ee, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/47c1aa8328774cc9b444a55b260ac65a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/fec6bde1efe545c78afa82ce91a88b28, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/41c5ac048b2647bf98b540e81ef29c44] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp, totalSize=47.4 K 2024-11-19T11:16:03,909 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:03,909 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. files: [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/23f42e682e144cb58a6d7e8500637630, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/9db11ed13f2f4d748d27ac4292f5bc8b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/deeae19b57ae44cd97003123c1213d72, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/b33883e363664d4c91f48f30545be293] 2024-11-19T11:16:03,909 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting be49d5608f5141e1875e91d308f5f2ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732014958297 2024-11-19T11:16:03,909 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23f42e682e144cb58a6d7e8500637630, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732014958297 2024-11-19T11:16:03,909 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 47c1aa8328774cc9b444a55b260ac65a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732014958439 2024-11-19T11:16:03,909 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9db11ed13f2f4d748d27ac4292f5bc8b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732014958439 2024-11-19T11:16:03,910 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting fec6bde1efe545c78afa82ce91a88b28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732014959616 2024-11-19T11:16:03,910 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 41c5ac048b2647bf98b540e81ef29c44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732014961921 2024-11-19T11:16:03,910 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting deeae19b57ae44cd97003123c1213d72, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732014959616 2024-11-19T11:16:03,910 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting b33883e363664d4c91f48f30545be293, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732014961920 2024-11-19T11:16:03,916 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:03,918 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d631ed8d9a4ea7a066117289c5e10132#B#compaction#329 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:03,918 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/ed39917aac88405e838612acd6aef325 is 50, key is test_row_0/B:col10/1732014961921/Put/seqid=0 2024-11-19T11:16:03,924 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119da2d18ad24084c36bed8a72559b44d7c_d631ed8d9a4ea7a066117289c5e10132 store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:03,926 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119da2d18ad24084c36bed8a72559b44d7c_d631ed8d9a4ea7a066117289c5e10132, store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:03,927 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119da2d18ad24084c36bed8a72559b44d7c_d631ed8d9a4ea7a066117289c5e10132 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:03,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742209_1385 (size=12493) 2024-11-19T11:16:03,939 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/ed39917aac88405e838612acd6aef325 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/ed39917aac88405e838612acd6aef325 2024-11-19T11:16:03,945 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d631ed8d9a4ea7a066117289c5e10132/B of d631ed8d9a4ea7a066117289c5e10132 into ed39917aac88405e838612acd6aef325(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:03,945 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:03,945 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., storeName=d631ed8d9a4ea7a066117289c5e10132/B, priority=12, startTime=1732014963907; duration=0sec 2024-11-19T11:16:03,945 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:03,945 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d631ed8d9a4ea7a066117289c5e10132:B 2024-11-19T11:16:03,945 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:16:03,947 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:16:03,947 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): d631ed8d9a4ea7a066117289c5e10132/C is initiating minor compaction (all files) 2024-11-19T11:16:03,947 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d631ed8d9a4ea7a066117289c5e10132/C in TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:03,947 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/599654f729614e758bfc36e2e7c3e55d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/c30b92b28d114cd3adc3fd648a7abb77, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/f3649235dcfb4d21b0cc2f36a56b2ba7, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/bf575832dd77437eb35a9edccfaff989] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp, totalSize=47.4 K 2024-11-19T11:16:03,948 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 599654f729614e758bfc36e2e7c3e55d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732014958297 2024-11-19T11:16:03,948 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting c30b92b28d114cd3adc3fd648a7abb77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732014958439 2024-11-19T11:16:03,949 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting f3649235dcfb4d21b0cc2f36a56b2ba7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732014959616 2024-11-19T11:16:03,949 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting bf575832dd77437eb35a9edccfaff989, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732014961921 2024-11-19T11:16:03,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T11:16:03,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:03,959 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing d631ed8d9a4ea7a066117289c5e10132 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-19T11:16:03,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=A 2024-11-19T11:16:03,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:03,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=B 2024-11-19T11:16:03,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:03,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=C 2024-11-19T11:16:03,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:03,963 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d631ed8d9a4ea7a066117289c5e10132#C#compaction#331 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:03,964 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/fcc8afb1852f4bafac7a0269c7389feb is 50, key is test_row_0/C:col10/1732014961921/Put/seqid=0 2024-11-19T11:16:03,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742210_1386 (size=4469) 2024-11-19T11:16:04,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742211_1387 (size=12493) 2024-11-19T11:16:04,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d00c853bdf714f8ca9a22545afb9d21c_d631ed8d9a4ea7a066117289c5e10132 is 50, key is test_row_0/A:col10/1732014962601/Put/seqid=0 2024-11-19T11:16:04,010 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/fcc8afb1852f4bafac7a0269c7389feb as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/fcc8afb1852f4bafac7a0269c7389feb 2024-11-19T11:16:04,017 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d631ed8d9a4ea7a066117289c5e10132/C of d631ed8d9a4ea7a066117289c5e10132 into fcc8afb1852f4bafac7a0269c7389feb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:04,017 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:04,017 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., storeName=d631ed8d9a4ea7a066117289c5e10132/C, priority=12, startTime=1732014963907; duration=0sec 2024-11-19T11:16:04,017 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:04,017 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d631ed8d9a4ea7a066117289c5e10132:C 2024-11-19T11:16:04,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742212_1388 (size=12304) 2024-11-19T11:16:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T11:16:04,367 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d631ed8d9a4ea7a066117289c5e10132#A#compaction#330 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:04,367 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/f9d0ace55dd540849de76728798fe2b1 is 175, key is test_row_0/A:col10/1732014961921/Put/seqid=0 2024-11-19T11:16:04,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742213_1389 (size=31447) 2024-11-19T11:16:04,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:04,433 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d00c853bdf714f8ca9a22545afb9d21c_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d00c853bdf714f8ca9a22545afb9d21c_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:04,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/7d505b1e177943c7b808a6b76d66f0d9, store: [table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:04,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/7d505b1e177943c7b808a6b76d66f0d9 is 175, key is test_row_0/A:col10/1732014962601/Put/seqid=0 2024-11-19T11:16:04,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742214_1390 (size=31105) 2024-11-19T11:16:04,777 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/f9d0ace55dd540849de76728798fe2b1 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/f9d0ace55dd540849de76728798fe2b1 2024-11-19T11:16:04,782 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d631ed8d9a4ea7a066117289c5e10132/A of d631ed8d9a4ea7a066117289c5e10132 into f9d0ace55dd540849de76728798fe2b1(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:04,782 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:04,782 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., storeName=d631ed8d9a4ea7a066117289c5e10132/A, priority=12, startTime=1732014963907; duration=0sec 2024-11-19T11:16:04,782 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:04,782 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d631ed8d9a4ea7a066117289c5e10132:A 2024-11-19T11:16:04,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:04,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:16:04,842 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/7d505b1e177943c7b808a6b76d66f0d9 2024-11-19T11:16:04,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732014964819, max timestamp=1732014964850Num uniques 859; 2024-11-19T11:16:04,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T11:16:04,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732014964819, max timestamp=1732014964850Num uniques 859; 2024-11-19T11:16:04,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T11:16:04,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732014964819, max timestamp=1732014964850Num uniques 859; 2024-11-19T11:16:04,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T11:16:04,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/93898131748343ba84b89dfb48a671b7 is 50, key is test_row_0/B:col10/1732014962601/Put/seqid=0 2024-11-19T11:16:04,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742215_1391 (size=12151) 2024-11-19T11:16:04,883 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/93898131748343ba84b89dfb48a671b7 2024-11-19T11:16:04,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/8ae7bc63c6e046aa9fd5b673defa5542 is 50, key is test_row_0/C:col10/1732014962601/Put/seqid=0 2024-11-19T11:16:04,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742216_1392 (size=12151) 2024-11-19T11:16:04,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:04,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015024903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:04,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:04,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:04,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:04,924 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/8ae7bc63c6e046aa9fd5b673defa5542 2024-11-19T11:16:04,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:04,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015024913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:04,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:04,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:04,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:04,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:04,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015024916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:04,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:04,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:04,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:04,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/7d505b1e177943c7b808a6b76d66f0d9 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/7d505b1e177943c7b808a6b76d66f0d9 2024-11-19T11:16:04,959 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/7d505b1e177943c7b808a6b76d66f0d9, entries=150, sequenceid=188, filesize=30.4 K 2024-11-19T11:16:04,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/93898131748343ba84b89dfb48a671b7 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/93898131748343ba84b89dfb48a671b7 2024-11-19T11:16:04,973 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/93898131748343ba84b89dfb48a671b7, entries=150, sequenceid=188, filesize=11.9 K 2024-11-19T11:16:04,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/8ae7bc63c6e046aa9fd5b673defa5542 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8ae7bc63c6e046aa9fd5b673defa5542 2024-11-19T11:16:04,978 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8ae7bc63c6e046aa9fd5b673defa5542, entries=150, sequenceid=188, filesize=11.9 K 2024-11-19T11:16:04,979 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for d631ed8d9a4ea7a066117289c5e10132 in 1020ms, sequenceid=188, compaction requested=false 2024-11-19T11:16:04,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:04,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:04,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-19T11:16:04,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-19T11:16:04,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-19T11:16:04,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7910 sec 2024-11-19T11:16:04,992 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.7960 sec 2024-11-19T11:16:05,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:05,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d631ed8d9a4ea7a066117289c5e10132 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-19T11:16:05,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=A 2024-11-19T11:16:05,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:05,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=B 2024-11-19T11:16:05,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:05,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=C 2024-11-19T11:16:05,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:05,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e34497f302ed4ffa986b41137e9a4c8c_d631ed8d9a4ea7a066117289c5e10132 is 50, key is test_row_0/A:col10/1732014964909/Put/seqid=0 2024-11-19T11:16:05,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:05,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015025065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:05,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:05,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:05,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015025067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:05,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:05,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:05,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:05,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:05,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:05,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015025076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:05,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:05,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:05,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:05,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742217_1393 (size=14794) 2024-11-19T11:16:05,093 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,099 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e34497f302ed4ffa986b41137e9a4c8c_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e34497f302ed4ffa986b41137e9a4c8c_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:05,101 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/74748d08055c440fb7af8f42da0eda34, store: [table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:05,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/74748d08055c440fb7af8f42da0eda34 is 175, key is test_row_0/A:col10/1732014964909/Put/seqid=0 2024-11-19T11:16:05,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742218_1394 (size=39749) 2024-11-19T11:16:05,162 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/74748d08055c440fb7af8f42da0eda34 2024-11-19T11:16:05,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/3dbba35b5c8d4955bd50b7682ec224d7 is 50, key is test_row_0/B:col10/1732014964909/Put/seqid=0 2024-11-19T11:16:05,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:05,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015025187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:05,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015025187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:05,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:05,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:05,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:05,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:05,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015025188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:05,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742219_1395 (size=12151) 2024-11-19T11:16:05,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/3dbba35b5c8d4955bd50b7682ec224d7 2024-11-19T11:16:05,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/42fb6c75891443eda454b118e269f5f4 is 50, key is test_row_0/C:col10/1732014964909/Put/seqid=0 2024-11-19T11:16:05,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742220_1396 (size=12151) 2024-11-19T11:16:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T11:16:05,349 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:16:05,350 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:16:05,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-19T11:16:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T11:16:05,352 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:16:05,353 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:16:05,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:16:05,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:05,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015025408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:05,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015025408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:05,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015025408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:05,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:05,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:05,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T11:16:05,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T11:16:05,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:05,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:16:05,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:05,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:05,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:05,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/42fb6c75891443eda454b118e269f5f4 2024-11-19T11:16:05,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/74748d08055c440fb7af8f42da0eda34 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/74748d08055c440fb7af8f42da0eda34 2024-11-19T11:16:05,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/74748d08055c440fb7af8f42da0eda34, entries=200, sequenceid=227, filesize=38.8 K 2024-11-19T11:16:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/3dbba35b5c8d4955bd50b7682ec224d7 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/3dbba35b5c8d4955bd50b7682ec224d7 2024-11-19T11:16:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/3dbba35b5c8d4955bd50b7682ec224d7, entries=150, sequenceid=227, filesize=11.9 K 2024-11-19T11:16:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/42fb6c75891443eda454b118e269f5f4 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/42fb6c75891443eda454b118e269f5f4 2024-11-19T11:16:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/42fb6c75891443eda454b118e269f5f4, entries=150, sequenceid=227, filesize=11.9 K 2024-11-19T11:16:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for d631ed8d9a4ea7a066117289c5e10132 in 624ms, sequenceid=227, compaction requested=true 2024-11-19T11:16:05,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:05,663 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d631ed8d9a4ea7a066117289c5e10132:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:16:05,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:05,664 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d631ed8d9a4ea7a066117289c5e10132:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:16:05,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:05,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d631ed8d9a4ea7a066117289c5e10132:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:16:05,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,665 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:05,665 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): d631ed8d9a4ea7a066117289c5e10132/A is initiating minor compaction (all files) 2024-11-19T11:16:05,665 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d631ed8d9a4ea7a066117289c5e10132/A in TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,665 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/f9d0ace55dd540849de76728798fe2b1, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/7d505b1e177943c7b808a6b76d66f0d9, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/74748d08055c440fb7af8f42da0eda34] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp, totalSize=99.9 K 2024-11-19T11:16:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,665 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:05,665 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. files: [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/f9d0ace55dd540849de76728798fe2b1, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/7d505b1e177943c7b808a6b76d66f0d9, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/74748d08055c440fb7af8f42da0eda34] 2024-11-19T11:16:05,665 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:05,665 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): d631ed8d9a4ea7a066117289c5e10132/B is initiating minor compaction (all files) 2024-11-19T11:16:05,665 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d631ed8d9a4ea7a066117289c5e10132/B in TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,665 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/ed39917aac88405e838612acd6aef325, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/93898131748343ba84b89dfb48a671b7, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/3dbba35b5c8d4955bd50b7682ec224d7] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp, totalSize=35.9 K 2024-11-19T11:16:05,666 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9d0ace55dd540849de76728798fe2b1, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732014961921 2024-11-19T11:16:05,666 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d505b1e177943c7b808a6b76d66f0d9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732014962597 2024-11-19T11:16:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,666 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74748d08055c440fb7af8f42da0eda34, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732014964891 2024-11-19T11:16:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T11:16:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,668 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting ed39917aac88405e838612acd6aef325, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732014961921 2024-11-19T11:16:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,668 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 93898131748343ba84b89dfb48a671b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732014962597 2024-11-19T11:16:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,669 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dbba35b5c8d4955bd50b7682ec224d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732014964891 2024-11-19T11:16:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T11:16:05,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:05,674 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing d631ed8d9a4ea7a066117289c5e10132 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T11:16:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=A 2024-11-19T11:16:05,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:05,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=B 2024-11-19T11:16:05,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:05,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=C 2024-11-19T11:16:05,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,675 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,679 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119734ddd02040f40e39679754089323578_d631ed8d9a4ea7a066117289c5e10132 store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,682 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119734ddd02040f40e39679754089323578_d631ed8d9a4ea7a066117289c5e10132, store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,682 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119734ddd02040f40e39679754089323578_d631ed8d9a4ea7a066117289c5e10132 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,686 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d631ed8d9a4ea7a066117289c5e10132#B#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:05,687 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/103ff3de3df14512bff746253ff91308 is 50, key is test_row_0/B:col10/1732014964909/Put/seqid=0 2024-11-19T11:16:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119cb241b64e2fe4f0ba181b2f541061722_d631ed8d9a4ea7a066117289c5e10132 is 50, key is test_row_1/A:col10/1732014965056/Put/seqid=0 2024-11-19T11:16:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742221_1397 (size=4469) 2024-11-19T11:16:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,714 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d631ed8d9a4ea7a066117289c5e10132#A#compaction#338 average throughput is 0.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,714 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/b4f86a7c7f44448798c5cb75bc0d794c is 175, key is test_row_0/A:col10/1732014964909/Put/seqid=0 2024-11-19T11:16:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742222_1398 (size=12595) 2024-11-19T11:16:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742224_1400 (size=31549) 2024-11-19T11:16:05,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:16:05,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,794 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/b4f86a7c7f44448798c5cb75bc0d794c as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/b4f86a7c7f44448798c5cb75bc0d794c 2024-11-19T11:16:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742223_1399 (size=9814) 2024-11-19T11:16:05,800 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d631ed8d9a4ea7a066117289c5e10132/A of d631ed8d9a4ea7a066117289c5e10132 into b4f86a7c7f44448798c5cb75bc0d794c(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:05,800 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:05,800 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., storeName=d631ed8d9a4ea7a066117289c5e10132/A, priority=13, startTime=1732014965663; duration=0sec 2024-11-19T11:16:05,800 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:05,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,800 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d631ed8d9a4ea7a066117289c5e10132:A 2024-11-19T11:16:05,801 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,804 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:05,804 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): d631ed8d9a4ea7a066117289c5e10132/C is initiating minor compaction (all files) 2024-11-19T11:16:05,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,804 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d631ed8d9a4ea7a066117289c5e10132/C in TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:05,804 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/fcc8afb1852f4bafac7a0269c7389feb, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8ae7bc63c6e046aa9fd5b673defa5542, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/42fb6c75891443eda454b118e269f5f4] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp, totalSize=35.9 K 2024-11-19T11:16:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,810 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcc8afb1852f4bafac7a0269c7389feb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732014961921 2024-11-19T11:16:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,811 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ae7bc63c6e046aa9fd5b673defa5542, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732014962597 2024-11-19T11:16:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,812 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42fb6c75891443eda454b118e269f5f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732014964891 2024-11-19T11:16:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:05,813 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119cb241b64e2fe4f0ba181b2f541061722_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119cb241b64e2fe4f0ba181b2f541061722_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:05,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/287dc1d624d24625aa492d35bd96f350, store: [table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:05,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/287dc1d624d24625aa492d35bd96f350 is 175, key is test_row_1/A:col10/1732014965056/Put/seqid=0 2024-11-19T11:16:05,848 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d631ed8d9a4ea7a066117289c5e10132#C#compaction#341 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:05,849 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/0f74082d04e64f4ebb47f2a4ea77c222 is 50, key is test_row_0/C:col10/1732014964909/Put/seqid=0 2024-11-19T11:16:05,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742225_1401 (size=22461) 2024-11-19T11:16:05,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742226_1402 (size=12595) 2024-11-19T11:16:05,891 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/0f74082d04e64f4ebb47f2a4ea77c222 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/0f74082d04e64f4ebb47f2a4ea77c222 2024-11-19T11:16:05,898 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d631ed8d9a4ea7a066117289c5e10132/C of d631ed8d9a4ea7a066117289c5e10132 into 0f74082d04e64f4ebb47f2a4ea77c222(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:05,898 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:05,898 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., storeName=d631ed8d9a4ea7a066117289c5e10132/C, priority=13, startTime=1732014965664; duration=0sec 2024-11-19T11:16:05,899 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:05,899 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d631ed8d9a4ea7a066117289c5e10132:C 2024-11-19T11:16:05,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732014965877, max timestamp=1732014965948Num uniques 859; 2024-11-19T11:16:05,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T11:16:05,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732014965877, max timestamp=1732014965948Num uniques 859; 2024-11-19T11:16:05,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T11:16:05,954 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732014965877, max timestamp=1732014965948Num uniques 859; 2024-11-19T11:16:05,954 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T11:16:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T11:16:06,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732014965973, max timestamp=1732014966044Num uniques 1068; 2024-11-19T11:16:06,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732014965973, max timestamp=1732014966044Num uniques 1068; 2024-11-19T11:16:06,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-19T11:16:06,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732014965973, max timestamp=1732014966044Num uniques 1068; 2024-11-19T11:16:06,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-19T11:16:06,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-19T11:16:06,147 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/103ff3de3df14512bff746253ff91308 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/103ff3de3df14512bff746253ff91308 2024-11-19T11:16:06,152 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d631ed8d9a4ea7a066117289c5e10132/B of d631ed8d9a4ea7a066117289c5e10132 into 103ff3de3df14512bff746253ff91308(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:06,153 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:06,153 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., storeName=d631ed8d9a4ea7a066117289c5e10132/B, priority=13, startTime=1732014965664; duration=0sec 2024-11-19T11:16:06,153 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:06,153 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d631ed8d9a4ea7a066117289c5e10132:B 2024-11-19T11:16:06,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732014966095, max timestamp=1732014966170Num uniques 1177; 2024-11-19T11:16:06,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-11-19T11:16:06,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732014966095, max timestamp=1732014966170Num uniques 1177; 2024-11-19T11:16:06,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-11-19T11:16:06,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732014966095, max timestamp=1732014966170Num uniques 1177; 2024-11-19T11:16:06,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-11-19T11:16:06,263 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/287dc1d624d24625aa492d35bd96f350 2024-11-19T11:16:06,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/2b20a60bf6fb493485fbffdfd848aca6 is 50, key is test_row_1/B:col10/1732014965056/Put/seqid=0 2024-11-19T11:16:06,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015026256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:06,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:06,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:06,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:06,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015026260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:06,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:06,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:06,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:06,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015026262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:06,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:06,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:06,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:06,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742227_1403 (size=9757) 2024-11-19T11:16:06,303 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/2b20a60bf6fb493485fbffdfd848aca6 2024-11-19T11:16:06,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/4dbe9f64eb1047c0ab95ecb3e2f9b0b8 is 50, key is test_row_1/C:col10/1732014965056/Put/seqid=0 2024-11-19T11:16:06,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742228_1404 (size=9757) 2024-11-19T11:16:06,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015026380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:06,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:06,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:06,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:06,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015026388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:06,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:06,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:06,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:06,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015026388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T11:16:06,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015026607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:06,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015026609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:06,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:06,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:06,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:06,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015026609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:06,766 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/4dbe9f64eb1047c0ab95ecb3e2f9b0b8 2024-11-19T11:16:06,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/287dc1d624d24625aa492d35bd96f350 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/287dc1d624d24625aa492d35bd96f350 2024-11-19T11:16:06,781 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/287dc1d624d24625aa492d35bd96f350, entries=100, sequenceid=235, filesize=21.9 K 2024-11-19T11:16:06,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/2b20a60bf6fb493485fbffdfd848aca6 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/2b20a60bf6fb493485fbffdfd848aca6 2024-11-19T11:16:06,791 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/2b20a60bf6fb493485fbffdfd848aca6, entries=100, sequenceid=235, filesize=9.5 K 2024-11-19T11:16:06,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/4dbe9f64eb1047c0ab95ecb3e2f9b0b8 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/4dbe9f64eb1047c0ab95ecb3e2f9b0b8 2024-11-19T11:16:06,797 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/4dbe9f64eb1047c0ab95ecb3e2f9b0b8, entries=100, sequenceid=235, filesize=9.5 K 2024-11-19T11:16:06,797 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for d631ed8d9a4ea7a066117289c5e10132 in 1124ms, sequenceid=235, compaction requested=false 2024-11-19T11:16:06,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:06,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:06,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-19T11:16:06,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-19T11:16:06,807 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-19T11:16:06,807 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4500 sec 2024-11-19T11:16:06,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.4580 sec 2024-11-19T11:16:06,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732014966254, max timestamp=1732014966933Num uniques 1136; 2024-11-19T11:16:06,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-19T11:16:06,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732014966254, max timestamp=1732014966933Num uniques 1136; 2024-11-19T11:16:06,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-19T11:16:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:06,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d631ed8d9a4ea7a066117289c5e10132 3/3 column families, dataSize=69.33 KB heapSize=187.27 KB 2024-11-19T11:16:06,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=A 2024-11-19T11:16:06,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:06,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=B 2024-11-19T11:16:06,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:06,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=C 2024-11-19T11:16:06,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T11:16:06,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199b6ff2af9ce14c958b6e6f2d583a85be_d631ed8d9a4ea7a066117289c5e10132 is 50, key is test_row_0/A:col10/1732014966933/Put/seqid=0 2024-11-19T11:16:07,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742229_1405 (size=14994) 2024-11-19T11:16:07,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014967020, max timestamp=1732014967057Num uniques 859; 2024-11-19T11:16:07,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:07,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014967020, max timestamp=1732014967057Num uniques 859; 2024-11-19T11:16:07,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:07,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014967020, max timestamp=1732014967057Num uniques 859; 2024-11-19T11:16:07,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:07,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015027159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:07,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015027159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:07,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:07,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:07,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015027160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:07,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015027270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:07,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015027278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:07,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:07,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:07,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015027287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:07,404 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:07,407 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199b6ff2af9ce14c958b6e6f2d583a85be_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199b6ff2af9ce14c958b6e6f2d583a85be_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:07,408 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/27b59f3f15e94c10918e4590afebc06d, store: [table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:07,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/27b59f3f15e94c10918e4590afebc06d is 175, key is test_row_0/A:col10/1732014966933/Put/seqid=0 2024-11-19T11:16:07,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742230_1406 (size=39949) 2024-11-19T11:16:07,437 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=310, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/27b59f3f15e94c10918e4590afebc06d 2024-11-19T11:16:07,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/9761fddb57ab42e99f66bfab8bed597b is 50, key is test_row_0/B:col10/1732014966933/Put/seqid=0 2024-11-19T11:16:07,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T11:16:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015027498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:07,508 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:16:07,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:07,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015027499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:07,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742231_1407 (size=12301) 2024-11-19T11:16:07,509 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:16:07,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-19T11:16:07,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/9761fddb57ab42e99f66bfab8bed597b 2024-11-19T11:16:07,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T11:16:07,511 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:16:07,512 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:16:07,513 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:16:07,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015027508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:07,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/36a95b41dc8248c2b5af40e33e54e625 is 50, key is test_row_0/C:col10/1732014966933/Put/seqid=0 2024-11-19T11:16:07,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742232_1408 (size=12301) 2024-11-19T11:16:07,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/36a95b41dc8248c2b5af40e33e54e625 2024-11-19T11:16:07,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/27b59f3f15e94c10918e4590afebc06d as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/27b59f3f15e94c10918e4590afebc06d 2024-11-19T11:16:07,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/27b59f3f15e94c10918e4590afebc06d, entries=200, sequenceid=310, filesize=39.0 K 2024-11-19T11:16:07,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/9761fddb57ab42e99f66bfab8bed597b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/9761fddb57ab42e99f66bfab8bed597b 2024-11-19T11:16:07,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/9761fddb57ab42e99f66bfab8bed597b, entries=150, sequenceid=310, filesize=12.0 K 2024-11-19T11:16:07,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/36a95b41dc8248c2b5af40e33e54e625 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/36a95b41dc8248c2b5af40e33e54e625 2024-11-19T11:16:07,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T11:16:07,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/36a95b41dc8248c2b5af40e33e54e625, entries=150, sequenceid=310, filesize=12.0 K 2024-11-19T11:16:07,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~69.33 KB/70990, heapSize ~187.22 KB/191712, currentSize=114.05 KB/116790 for d631ed8d9a4ea7a066117289c5e10132 in 695ms, sequenceid=310, compaction requested=true 2024-11-19T11:16:07,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:07,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d631ed8d9a4ea7a066117289c5e10132:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:16:07,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:07,638 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:07,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d631ed8d9a4ea7a066117289c5e10132:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:16:07,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:07,639 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:07,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d631ed8d9a4ea7a066117289c5e10132:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:16:07,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:07,640 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:07,640 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): d631ed8d9a4ea7a066117289c5e10132/A is initiating minor compaction (all files) 2024-11-19T11:16:07,640 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d631ed8d9a4ea7a066117289c5e10132/A in TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:07,640 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/b4f86a7c7f44448798c5cb75bc0d794c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/287dc1d624d24625aa492d35bd96f350, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/27b59f3f15e94c10918e4590afebc06d] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp, totalSize=91.8 K 2024-11-19T11:16:07,640 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:07,640 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. files: [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/b4f86a7c7f44448798c5cb75bc0d794c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/287dc1d624d24625aa492d35bd96f350, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/27b59f3f15e94c10918e4590afebc06d] 2024-11-19T11:16:07,641 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:07,641 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): d631ed8d9a4ea7a066117289c5e10132/B is initiating minor compaction (all files) 2024-11-19T11:16:07,641 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d631ed8d9a4ea7a066117289c5e10132/B in TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:07,641 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/103ff3de3df14512bff746253ff91308, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/2b20a60bf6fb493485fbffdfd848aca6, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/9761fddb57ab42e99f66bfab8bed597b] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp, totalSize=33.8 K 2024-11-19T11:16:07,641 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4f86a7c7f44448798c5cb75bc0d794c, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732014964891 2024-11-19T11:16:07,644 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 103ff3de3df14512bff746253ff91308, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732014964891 2024-11-19T11:16:07,644 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 287dc1d624d24625aa492d35bd96f350, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732014965056 2024-11-19T11:16:07,645 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b20a60bf6fb493485fbffdfd848aca6, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732014965056 2024-11-19T11:16:07,645 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27b59f3f15e94c10918e4590afebc06d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732014966254 2024-11-19T11:16:07,648 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 9761fddb57ab42e99f66bfab8bed597b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732014966254 2024-11-19T11:16:07,666 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d631ed8d9a4ea7a066117289c5e10132#B#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:07,667 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/0b02d7cbcb2347a3bdca3d4f0daab65b is 50, key is test_row_0/B:col10/1732014966933/Put/seqid=0 2024-11-19T11:16:07,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T11:16:07,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:07,669 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing d631ed8d9a4ea7a066117289c5e10132 3/3 column families, dataSize=114.05 KB heapSize=325.22 KB 2024-11-19T11:16:07,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=A 2024-11-19T11:16:07,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:07,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=B 2024-11-19T11:16:07,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:07,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=C 2024-11-19T11:16:07,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:07,673 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:07,697 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411192a6175d082d149a39dfc4d52bb3490b7_d631ed8d9a4ea7a066117289c5e10132 store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:07,699 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411192a6175d082d149a39dfc4d52bb3490b7_d631ed8d9a4ea7a066117289c5e10132, store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:07,700 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192a6175d082d149a39dfc4d52bb3490b7_d631ed8d9a4ea7a066117289c5e10132 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:07,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742233_1409 (size=12847) 2024-11-19T11:16:07,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a1ac451156a64430927ed3c71ff535e4_d631ed8d9a4ea7a066117289c5e10132 is 50, key is test_row_0/A:col10/1732014967140/Put/seqid=0 2024-11-19T11:16:07,750 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/0b02d7cbcb2347a3bdca3d4f0daab65b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/0b02d7cbcb2347a3bdca3d4f0daab65b 2024-11-19T11:16:07,757 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d631ed8d9a4ea7a066117289c5e10132/B of d631ed8d9a4ea7a066117289c5e10132 into 0b02d7cbcb2347a3bdca3d4f0daab65b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:07,757 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:07,757 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., storeName=d631ed8d9a4ea7a066117289c5e10132/B, priority=13, startTime=1732014967638; duration=0sec 2024-11-19T11:16:07,758 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:07,758 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d631ed8d9a4ea7a066117289c5e10132:B 2024-11-19T11:16:07,758 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:07,759 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:07,759 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): d631ed8d9a4ea7a066117289c5e10132/C is initiating minor compaction (all files) 2024-11-19T11:16:07,759 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d631ed8d9a4ea7a066117289c5e10132/C in TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:07,759 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/0f74082d04e64f4ebb47f2a4ea77c222, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/4dbe9f64eb1047c0ab95ecb3e2f9b0b8, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/36a95b41dc8248c2b5af40e33e54e625] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp, totalSize=33.8 K 2024-11-19T11:16:07,760 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f74082d04e64f4ebb47f2a4ea77c222, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732014964891 2024-11-19T11:16:07,760 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dbe9f64eb1047c0ab95ecb3e2f9b0b8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732014965056 2024-11-19T11:16:07,760 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 36a95b41dc8248c2b5af40e33e54e625, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732014966254 2024-11-19T11:16:07,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742234_1410 (size=4469) 2024-11-19T11:16:07,775 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d631ed8d9a4ea7a066117289c5e10132#A#compaction#348 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:07,775 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/d8b12103057a44e1910948d3bcd56f5c is 175, key is test_row_0/A:col10/1732014966933/Put/seqid=0 2024-11-19T11:16:07,784 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d631ed8d9a4ea7a066117289c5e10132#C#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:07,784 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/35333dd8c46440d780894eb0af3d4041 is 50, key is test_row_0/C:col10/1732014966933/Put/seqid=0 2024-11-19T11:16:07,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742235_1411 (size=12454) 2024-11-19T11:16:07,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T11:16:07,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:16:07,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:07,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742236_1412 (size=31801) 2024-11-19T11:16:07,878 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/d8b12103057a44e1910948d3bcd56f5c as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/d8b12103057a44e1910948d3bcd56f5c 2024-11-19T11:16:07,883 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d631ed8d9a4ea7a066117289c5e10132/A of d631ed8d9a4ea7a066117289c5e10132 into d8b12103057a44e1910948d3bcd56f5c(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:07,883 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:07,883 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., storeName=d631ed8d9a4ea7a066117289c5e10132/A, priority=13, startTime=1732014967638; duration=0sec 2024-11-19T11:16:07,883 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:07,883 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d631ed8d9a4ea7a066117289c5e10132:A 2024-11-19T11:16:07,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742237_1413 (size=12847) 2024-11-19T11:16:07,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:07,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:07,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015027868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:07,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015027870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:07,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:07,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015027888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:07,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:07,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:08,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:08,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015027998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:08,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:08,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:08,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015027998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:08,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:08,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015027998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:08,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T11:16:08,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:08,230 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a1ac451156a64430927ed3c71ff535e4_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a1ac451156a64430927ed3c71ff535e4_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:08,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:08,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015028217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:08,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/4ec2bf5b61bb488b87b362dd834c7dbb, store: [table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:08,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:08,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:08,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/4ec2bf5b61bb488b87b362dd834c7dbb is 175, key is test_row_0/A:col10/1732014967140/Put/seqid=0 2024-11-19T11:16:08,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015028217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:08,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:08,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:08,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015028217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:08,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742238_1414 (size=31255) 2024-11-19T11:16:08,264 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=344, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/4ec2bf5b61bb488b87b362dd834c7dbb 2024-11-19T11:16:08,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/6126ca3080744eb09b0261c973fcab69 is 50, key is test_row_0/B:col10/1732014967140/Put/seqid=0 2024-11-19T11:16:08,293 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/35333dd8c46440d780894eb0af3d4041 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/35333dd8c46440d780894eb0af3d4041 2024-11-19T11:16:08,299 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d631ed8d9a4ea7a066117289c5e10132/C of d631ed8d9a4ea7a066117289c5e10132 into 35333dd8c46440d780894eb0af3d4041(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:08,299 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:08,299 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., storeName=d631ed8d9a4ea7a066117289c5e10132/C, priority=13, startTime=1732014967639; duration=0sec 2024-11-19T11:16:08,299 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:08,299 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d631ed8d9a4ea7a066117289c5e10132:C 2024-11-19T11:16:08,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742239_1415 (size=12301) 2024-11-19T11:16:08,334 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/6126ca3080744eb09b0261c973fcab69 2024-11-19T11:16:08,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/50a762c0cd9f478f942a8610d4927c49 is 50, key is test_row_0/C:col10/1732014967140/Put/seqid=0 2024-11-19T11:16:08,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742240_1416 (size=12301) 2024-11-19T11:16:08,381 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/50a762c0cd9f478f942a8610d4927c49 2024-11-19T11:16:08,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/4ec2bf5b61bb488b87b362dd834c7dbb as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/4ec2bf5b61bb488b87b362dd834c7dbb 2024-11-19T11:16:08,398 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/4ec2bf5b61bb488b87b362dd834c7dbb, entries=150, sequenceid=344, filesize=30.5 K 2024-11-19T11:16:08,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/6126ca3080744eb09b0261c973fcab69 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/6126ca3080744eb09b0261c973fcab69 2024-11-19T11:16:08,403 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/6126ca3080744eb09b0261c973fcab69, entries=150, sequenceid=344, filesize=12.0 K 2024-11-19T11:16:08,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/50a762c0cd9f478f942a8610d4927c49 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/50a762c0cd9f478f942a8610d4927c49 2024-11-19T11:16:08,408 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/50a762c0cd9f478f942a8610d4927c49, entries=150, sequenceid=344, filesize=12.0 K 2024-11-19T11:16:08,409 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~325.17 KB/332976, currentSize=67.09 KB/68700 for d631ed8d9a4ea7a066117289c5e10132 in 741ms, sequenceid=344, compaction requested=false 2024-11-19T11:16:08,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:08,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:08,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-19T11:16:08,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-19T11:16:08,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-19T11:16:08,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 897 msec 2024-11-19T11:16:08,414 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 904 msec 2024-11-19T11:16:08,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:08,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d631ed8d9a4ea7a066117289c5e10132 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T11:16:08,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=A 2024-11-19T11:16:08,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:08,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=B 2024-11-19T11:16:08,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:08,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=C 2024-11-19T11:16:08,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:08,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ea506d1e8c7a48848ad4bc5d9a23149d_d631ed8d9a4ea7a066117289c5e10132 is 50, key is test_row_0/A:col10/1732014968540/Put/seqid=0 2024-11-19T11:16:08,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742241_1417 (size=14994) 2024-11-19T11:16:08,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:08,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015028580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:08,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:08,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015028586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:08,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:08,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:08,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:08,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015028594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:08,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:08,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T11:16:08,647 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:16:08,647 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:16:08,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-19T11:16:08,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T11:16:08,649 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:16:08,649 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:16:08,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:16:08,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:08,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015028697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:08,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:08,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015028698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:08,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:08,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:08,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015028708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:08,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T11:16:08,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T11:16:08,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:08,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:16:08,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:08,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:08,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:08,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015028918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:08,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:08,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015028918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015028918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:08,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T11:16:08,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:08,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:16:08,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:08,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:08,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:08,966 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T11:16:08,973 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ea506d1e8c7a48848ad4bc5d9a23149d_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ea506d1e8c7a48848ad4bc5d9a23149d_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:08,977 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/d83e75c2f6714cf9b8dc614d02a36727, store: [table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:08,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/d83e75c2f6714cf9b8dc614d02a36727 is 175, key is test_row_0/A:col10/1732014968540/Put/seqid=0 2024-11-19T11:16:08,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742242_1418 (size=39949) 2024-11-19T11:16:08,998 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=361, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/d83e75c2f6714cf9b8dc614d02a36727 2024-11-19T11:16:09,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/8a872aa916d948c6aceb44ff198a517c is 50, key is test_row_0/B:col10/1732014968540/Put/seqid=0 2024-11-19T11:16:09,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742243_1419 (size=12301) 2024-11-19T11:16:09,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/8a872aa916d948c6aceb44ff198a517c 2024-11-19T11:16:09,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/5a909447dee348f4b11f640c0a24c6f0 is 50, key is test_row_0/C:col10/1732014968540/Put/seqid=0 2024-11-19T11:16:09,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742244_1420 (size=12301) 2024-11-19T11:16:09,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/5a909447dee348f4b11f640c0a24c6f0 2024-11-19T11:16:09,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/d83e75c2f6714cf9b8dc614d02a36727 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/d83e75c2f6714cf9b8dc614d02a36727 2024-11-19T11:16:09,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/d83e75c2f6714cf9b8dc614d02a36727, entries=200, sequenceid=361, filesize=39.0 K 2024-11-19T11:16:09,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/8a872aa916d948c6aceb44ff198a517c as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/8a872aa916d948c6aceb44ff198a517c 2024-11-19T11:16:09,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/8a872aa916d948c6aceb44ff198a517c, entries=150, sequenceid=361, filesize=12.0 K 2024-11-19T11:16:09,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/5a909447dee348f4b11f640c0a24c6f0 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/5a909447dee348f4b11f640c0a24c6f0 2024-11-19T11:16:09,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/5a909447dee348f4b11f640c0a24c6f0, entries=150, sequenceid=361, filesize=12.0 K 2024-11-19T11:16:09,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d631ed8d9a4ea7a066117289c5e10132 in 538ms, sequenceid=361, compaction requested=true 2024-11-19T11:16:09,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:09,079 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:09,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d631ed8d9a4ea7a066117289c5e10132:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:16:09,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:09,079 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:09,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d631ed8d9a4ea7a066117289c5e10132:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:16:09,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:09,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d631ed8d9a4ea7a066117289c5e10132:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:16:09,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:09,081 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:09,081 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:09,081 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): d631ed8d9a4ea7a066117289c5e10132/B is initiating minor compaction (all files) 2024-11-19T11:16:09,081 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): d631ed8d9a4ea7a066117289c5e10132/A is initiating minor compaction (all files) 2024-11-19T11:16:09,081 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d631ed8d9a4ea7a066117289c5e10132/B in TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:09,081 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d631ed8d9a4ea7a066117289c5e10132/A in TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:09,081 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/d8b12103057a44e1910948d3bcd56f5c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/4ec2bf5b61bb488b87b362dd834c7dbb, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/d83e75c2f6714cf9b8dc614d02a36727] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp, totalSize=100.6 K 2024-11-19T11:16:09,081 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/0b02d7cbcb2347a3bdca3d4f0daab65b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/6126ca3080744eb09b0261c973fcab69, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/8a872aa916d948c6aceb44ff198a517c] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp, totalSize=36.6 K 2024-11-19T11:16:09,081 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:09,081 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. files: [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/d8b12103057a44e1910948d3bcd56f5c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/4ec2bf5b61bb488b87b362dd834c7dbb, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/d83e75c2f6714cf9b8dc614d02a36727] 2024-11-19T11:16:09,086 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8b12103057a44e1910948d3bcd56f5c, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732014966254 2024-11-19T11:16:09,086 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b02d7cbcb2347a3bdca3d4f0daab65b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732014966254 2024-11-19T11:16:09,086 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ec2bf5b61bb488b87b362dd834c7dbb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732014967140 2024-11-19T11:16:09,086 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 6126ca3080744eb09b0261c973fcab69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732014967140 2024-11-19T11:16:09,086 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting d83e75c2f6714cf9b8dc614d02a36727, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732014967828 2024-11-19T11:16:09,086 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a872aa916d948c6aceb44ff198a517c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732014967866 2024-11-19T11:16:09,097 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d631ed8d9a4ea7a066117289c5e10132#B#compaction#356 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:09,098 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/c0722578b00d4ea1bf301580975ea6d7 is 50, key is test_row_0/B:col10/1732014968540/Put/seqid=0 2024-11-19T11:16:09,105 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:09,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T11:16:09,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:09,108 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing d631ed8d9a4ea7a066117289c5e10132 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T11:16:09,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=A 2024-11-19T11:16:09,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:09,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=B 2024-11-19T11:16:09,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:09,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=C 2024-11-19T11:16:09,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:09,116 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411199c32759682ba420e9aaf3c9c4ee40903_d631ed8d9a4ea7a066117289c5e10132 store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:09,118 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411199c32759682ba420e9aaf3c9c4ee40903_d631ed8d9a4ea7a066117289c5e10132, store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:09,118 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199c32759682ba420e9aaf3c9c4ee40903_d631ed8d9a4ea7a066117289c5e10132 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:09,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742245_1421 (size=12949) 2024-11-19T11:16:09,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bb7134ef809142e8baf23f48e0aa2ee5_d631ed8d9a4ea7a066117289c5e10132 is 50, key is test_row_0/A:col10/1732014968579/Put/seqid=0 2024-11-19T11:16:09,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742246_1422 (size=4469) 2024-11-19T11:16:09,158 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d631ed8d9a4ea7a066117289c5e10132#A#compaction#357 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:09,159 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/3b1dd4a4284a4f1a8459d0cbcaf090e4 is 175, key is test_row_0/A:col10/1732014968540/Put/seqid=0 2024-11-19T11:16:09,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742247_1423 (size=12454) 2024-11-19T11:16:09,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742248_1424 (size=31903) 2024-11-19T11:16:09,182 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/3b1dd4a4284a4f1a8459d0cbcaf090e4 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/3b1dd4a4284a4f1a8459d0cbcaf090e4 2024-11-19T11:16:09,188 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d631ed8d9a4ea7a066117289c5e10132/A of d631ed8d9a4ea7a066117289c5e10132 into 3b1dd4a4284a4f1a8459d0cbcaf090e4(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:09,188 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:09,188 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., storeName=d631ed8d9a4ea7a066117289c5e10132/A, priority=13, startTime=1732014969079; duration=0sec 2024-11-19T11:16:09,188 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:09,188 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d631ed8d9a4ea7a066117289c5e10132:A 2024-11-19T11:16:09,188 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:09,189 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:09,189 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): d631ed8d9a4ea7a066117289c5e10132/C is initiating minor compaction (all files) 2024-11-19T11:16:09,190 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d631ed8d9a4ea7a066117289c5e10132/C in TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:09,190 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/35333dd8c46440d780894eb0af3d4041, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/50a762c0cd9f478f942a8610d4927c49, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/5a909447dee348f4b11f640c0a24c6f0] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp, totalSize=36.6 K 2024-11-19T11:16:09,190 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35333dd8c46440d780894eb0af3d4041, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732014966254 2024-11-19T11:16:09,190 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50a762c0cd9f478f942a8610d4927c49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732014967140 2024-11-19T11:16:09,191 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a909447dee348f4b11f640c0a24c6f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732014967866 2024-11-19T11:16:09,205 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d631ed8d9a4ea7a066117289c5e10132#C#compaction#359 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:09,206 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/8ce4183aabfe4afbac6a38cb1fea5065 is 50, key is test_row_0/C:col10/1732014968540/Put/seqid=0 2024-11-19T11:16:09,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742249_1425 (size=12949) 2024-11-19T11:16:09,227 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/8ce4183aabfe4afbac6a38cb1fea5065 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8ce4183aabfe4afbac6a38cb1fea5065 2024-11-19T11:16:09,232 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d631ed8d9a4ea7a066117289c5e10132/C of d631ed8d9a4ea7a066117289c5e10132 into 8ce4183aabfe4afbac6a38cb1fea5065(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:09,232 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:09,232 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., storeName=d631ed8d9a4ea7a066117289c5e10132/C, priority=13, startTime=1732014969079; duration=0sec 2024-11-19T11:16:09,232 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:09,232 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d631ed8d9a4ea7a066117289c5e10132:C 2024-11-19T11:16:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:09,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:16:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T11:16:09,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015029293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:09,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015029294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:09,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:09,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015029297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:09,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:09,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015029407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:09,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:09,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015029417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:09,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015029417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:09,534 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/c0722578b00d4ea1bf301580975ea6d7 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/c0722578b00d4ea1bf301580975ea6d7 2024-11-19T11:16:09,537 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d631ed8d9a4ea7a066117289c5e10132/B of d631ed8d9a4ea7a066117289c5e10132 into c0722578b00d4ea1bf301580975ea6d7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:09,538 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:09,538 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., storeName=d631ed8d9a4ea7a066117289c5e10132/B, priority=13, startTime=1732014969079; duration=0sec 2024-11-19T11:16:09,538 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:09,538 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d631ed8d9a4ea7a066117289c5e10132:B 2024-11-19T11:16:09,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:09,578 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bb7134ef809142e8baf23f48e0aa2ee5_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bb7134ef809142e8baf23f48e0aa2ee5_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:09,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/17952deb21a34760906fcfe40f0e4d83, store: [table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:09,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/17952deb21a34760906fcfe40f0e4d83 is 175, key is test_row_0/A:col10/1732014968579/Put/seqid=0 2024-11-19T11:16:09,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742250_1426 (size=31255) 2024-11-19T11:16:09,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:09,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015029617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:09,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:09,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:09,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015029637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:09,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:09,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:09,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015029637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:09,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:09,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T11:16:09,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:09,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015029928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:09,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:09,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:09,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015029947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:09,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:09,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015029947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:09,983 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=379, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/17952deb21a34760906fcfe40f0e4d83 2024-11-19T11:16:09,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/06e7a207f911499bb9f96d3d2e05b0cd is 50, key is test_row_0/B:col10/1732014968579/Put/seqid=0 2024-11-19T11:16:10,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742251_1427 (size=12301) 2024-11-19T11:16:10,006 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/06e7a207f911499bb9f96d3d2e05b0cd 2024-11-19T11:16:10,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/256934d1783144b4a9f9a7ada8b74f64 is 50, key is test_row_0/C:col10/1732014968579/Put/seqid=0 2024-11-19T11:16:10,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742252_1428 (size=12301) 2024-11-19T11:16:10,420 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/256934d1783144b4a9f9a7ada8b74f64 2024-11-19T11:16:10,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/17952deb21a34760906fcfe40f0e4d83 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/17952deb21a34760906fcfe40f0e4d83 2024-11-19T11:16:10,452 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/17952deb21a34760906fcfe40f0e4d83, entries=150, sequenceid=379, filesize=30.5 K 2024-11-19T11:16:10,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/06e7a207f911499bb9f96d3d2e05b0cd as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/06e7a207f911499bb9f96d3d2e05b0cd 2024-11-19T11:16:10,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015030447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:10,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:10,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:10,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:10,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015030458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:10,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015030458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:10,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:10,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:10,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:10,478 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/06e7a207f911499bb9f96d3d2e05b0cd, entries=150, sequenceid=379, filesize=12.0 K 2024-11-19T11:16:10,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/256934d1783144b4a9f9a7ada8b74f64 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/256934d1783144b4a9f9a7ada8b74f64 2024-11-19T11:16:10,484 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/256934d1783144b4a9f9a7ada8b74f64, entries=150, sequenceid=379, filesize=12.0 K 2024-11-19T11:16:10,485 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for d631ed8d9a4ea7a066117289c5e10132 in 1377ms, sequenceid=379, compaction requested=false 2024-11-19T11:16:10,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:10,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:10,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-19T11:16:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-19T11:16:10,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-19T11:16:10,499 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8380 sec 2024-11-19T11:16:10,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.8540 sec 2024-11-19T11:16:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T11:16:10,797 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:16:10,798 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:16:10,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-19T11:16:10,799 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:16:10,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T11:16:10,800 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:16:10,801 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:16:10,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T11:16:10,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T11:16:10,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:10,953 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing d631ed8d9a4ea7a066117289c5e10132 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T11:16:10,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=A 2024-11-19T11:16:10,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:10,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=B 2024-11-19T11:16:10,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:10,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=C 2024-11-19T11:16:10,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:10,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111919d1b0058885402fb6290a82bb98f036_d631ed8d9a4ea7a066117289c5e10132 is 50, key is test_row_0/A:col10/1732014969291/Put/seqid=0 2024-11-19T11:16:10,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742253_1429 (size=12454) 2024-11-19T11:16:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:10,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. as already flushing 2024-11-19T11:16:11,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732015031039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:11,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:11,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:11,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1732015031040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:11,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:11,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:11,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:11,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T11:16:11,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:11,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732015031147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:11,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:11,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1732015031147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:11,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:11,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:11,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:11,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:11,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:11,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:11,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732015031358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:11,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:11,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:11,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:11,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:11,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1732015031358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:11,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:11,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:11,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:11,366 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111919d1b0058885402fb6290a82bb98f036_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111919d1b0058885402fb6290a82bb98f036_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:11,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/391175da6d694506b2dc8f4f120c877c, store: [table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:11,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/391175da6d694506b2dc8f4f120c877c is 175, key is test_row_0/A:col10/1732014969291/Put/seqid=0 2024-11-19T11:16:11,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742254_1430 (size=31255) 2024-11-19T11:16:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T11:16:11,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:11,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57400 deadline: 1732015031467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:11,474 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_1' in d631ed8d9a4ea7a066117289c5e10132 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:11,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:11,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1732015031477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:11,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:11,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57390 deadline: 1732015031477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:11,485 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_2' in d631ed8d9a4ea7a066117289c5e10132 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:11,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:11,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:11,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:11,485 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_0' in d631ed8d9a4ea7a066117289c5e10132 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:11,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:11,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732015031668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:11,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:11,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:11,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:11,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:11,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1732015031677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:11,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:11,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:11,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:11,771 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=397, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/391175da6d694506b2dc8f4f120c877c 2024-11-19T11:16:11,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/f0116ce5607d4b18a2e0cbe630b9e1d4 is 50, key is test_row_0/B:col10/1732014969291/Put/seqid=0 2024-11-19T11:16:11,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742255_1431 (size=12301) 2024-11-19T11:16:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T11:16:12,179 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/f0116ce5607d4b18a2e0cbe630b9e1d4 2024-11-19T11:16:12,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/697766f3ac7d4613b577c8a854f4f618 is 50, key is test_row_0/C:col10/1732014969291/Put/seqid=0 2024-11-19T11:16:12,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732015032187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:12,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57364 deadline: 1732015032198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:12,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:12,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:12,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:12,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:12,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:12,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:12,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742256_1432 (size=12301) 2024-11-19T11:16:12,238 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/697766f3ac7d4613b577c8a854f4f618 2024-11-19T11:16:12,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/391175da6d694506b2dc8f4f120c877c as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/391175da6d694506b2dc8f4f120c877c 2024-11-19T11:16:12,248 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/391175da6d694506b2dc8f4f120c877c, entries=150, sequenceid=397, filesize=30.5 K 2024-11-19T11:16:12,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/f0116ce5607d4b18a2e0cbe630b9e1d4 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/f0116ce5607d4b18a2e0cbe630b9e1d4 2024-11-19T11:16:12,264 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/f0116ce5607d4b18a2e0cbe630b9e1d4, entries=150, sequenceid=397, filesize=12.0 K 2024-11-19T11:16:12,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/697766f3ac7d4613b577c8a854f4f618 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/697766f3ac7d4613b577c8a854f4f618 2024-11-19T11:16:12,269 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/697766f3ac7d4613b577c8a854f4f618, entries=150, sequenceid=397, filesize=12.0 K 2024-11-19T11:16:12,271 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d631ed8d9a4ea7a066117289c5e10132 in 1317ms, sequenceid=397, compaction requested=true 2024-11-19T11:16:12,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:12,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:12,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-19T11:16:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-19T11:16:12,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-19T11:16:12,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4720 sec 2024-11-19T11:16:12,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.4770 sec 2024-11-19T11:16:12,549 INFO [Thread-1579 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1579. 2024-11-19T11:16:12,549 DEBUG [Thread-1579 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:16:12,549 DEBUG [Thread-1579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:12,550 DEBUG [Thread-1579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:12,550 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:16:12,551 INFO [Thread-1583 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1583. 2024-11-19T11:16:12,552 DEBUG [Thread-1583 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:16:12,552 DEBUG [Thread-1583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:12,552 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:16:12,552 DEBUG [Thread-1583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:12,557 INFO [Thread-1577 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1577. 2024-11-19T11:16:12,557 DEBUG [Thread-1577 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:16:12,557 DEBUG [Thread-1577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:12,558 DEBUG [Thread-1577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:12,558 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:16:12,561 INFO [Thread-1585 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1585. 2024-11-19T11:16:12,561 DEBUG [Thread-1585 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:16:12,561 DEBUG [Thread-1585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:12,561 DEBUG [Thread-1585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:12,561 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:16:12,563 INFO [Thread-1581 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1581. 2024-11-19T11:16:12,563 DEBUG [Thread-1581 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:16:12,563 DEBUG [Thread-1581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:12,563 DEBUG [Thread-1581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:12,563 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:16:12,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T11:16:12,948 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:16:13,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:13,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d631ed8d9a4ea7a066117289c5e10132 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T11:16:13,219 INFO [Thread-1566 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1566. 2024-11-19T11:16:13,219 INFO [Thread-1568 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1568. 2024-11-19T11:16:13,219 DEBUG [Thread-1566 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:16:13,219 DEBUG [Thread-1568 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:16:13,219 DEBUG [Thread-1566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:13,219 DEBUG [Thread-1568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:13,219 DEBUG [Thread-1568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:13,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=A 2024-11-19T11:16:13,219 DEBUG [Thread-1566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:13,219 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:16:13,219 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:16:13,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:13,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=B 2024-11-19T11:16:13,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:13,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=C 2024-11-19T11:16:13,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:13,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111962c30aab9eec41f7abc844db84484193_d631ed8d9a4ea7a066117289c5e10132 is 50, key is test_row_0/A:col10/1732014973218/Put/seqid=0 2024-11-19T11:16:13,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742257_1433 (size=12454) 2024-11-19T11:16:13,491 INFO [Thread-1574 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1574. 2024-11-19T11:16:13,491 DEBUG [Thread-1574 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:16:13,491 DEBUG [Thread-1574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:13,491 DEBUG [Thread-1574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:13,491 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:16:13,501 INFO [Thread-1572 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1572. 2024-11-19T11:16:13,501 INFO [Thread-1570 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1570. 2024-11-19T11:16:13,501 DEBUG [Thread-1572 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:16:13,501 DEBUG [Thread-1570 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:16:13,501 DEBUG [Thread-1572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:13,501 DEBUG [Thread-1570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:13,501 DEBUG [Thread-1570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:13,501 DEBUG [Thread-1572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:13,501 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:16:13,501 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:16:13,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T11:16:13,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-19T11:16:13,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-19T11:16:13,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-19T11:16:13,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-19T11:16:13,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-19T11:16:13,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T11:16:13,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T11:16:13,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1088 2024-11-19T11:16:13,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3258 rows 2024-11-19T11:16:13,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1077 2024-11-19T11:16:13,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3225 rows 2024-11-19T11:16:13,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1088 2024-11-19T11:16:13,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3263 rows 2024-11-19T11:16:13,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1084 2024-11-19T11:16:13,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3249 rows 2024-11-19T11:16:13,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1088 2024-11-19T11:16:13,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3257 rows 2024-11-19T11:16:13,503 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T11:16:13,503 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T11:16:13,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:13,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:13,503 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:16:13,505 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T11:16:13,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T11:16:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T11:16:13,510 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014973509"}]},"ts":"1732014973509"} 2024-11-19T11:16:13,511 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T11:16:13,511 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T11:16:13,511 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T11:16:13,512 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d631ed8d9a4ea7a066117289c5e10132, UNASSIGN}] 2024-11-19T11:16:13,513 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d631ed8d9a4ea7a066117289c5e10132, UNASSIGN 2024-11-19T11:16:13,513 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=d631ed8d9a4ea7a066117289c5e10132, regionState=CLOSING, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:16:13,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d631ed8d9a4ea7a066117289c5e10132, UNASSIGN because future has completed 2024-11-19T11:16:13,515 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T11:16:13,515 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640}] 2024-11-19T11:16:13,567 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T11:16:13,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T11:16:13,630 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:13,638 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111962c30aab9eec41f7abc844db84484193_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111962c30aab9eec41f7abc844db84484193_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:13,639 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/8612cefb3b6844a59872760a670ec125, store: [table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:13,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/8612cefb3b6844a59872760a670ec125 is 175, key is test_row_0/A:col10/1732014973218/Put/seqid=0 2024-11-19T11:16:13,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742258_1434 (size=31255) 2024-11-19T11:16:13,667 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:13,667 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T11:16:13,668 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing d631ed8d9a4ea7a066117289c5e10132, disabling compactions & flushes 2024-11-19T11:16:13,668 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:13,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T11:16:14,047 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=416, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/8612cefb3b6844a59872760a670ec125 2024-11-19T11:16:14,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/320846b6d4724ba9aa7552a7911e8acb is 50, key is test_row_0/B:col10/1732014973218/Put/seqid=0 2024-11-19T11:16:14,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742259_1435 (size=12301) 2024-11-19T11:16:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T11:16:14,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/320846b6d4724ba9aa7552a7911e8acb 2024-11-19T11:16:14,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/8cc2eafdea724f3db9de01e00f37a7a0 is 50, key is test_row_0/C:col10/1732014973218/Put/seqid=0 2024-11-19T11:16:14,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742260_1436 (size=12301) 2024-11-19T11:16:14,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T11:16:14,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/8cc2eafdea724f3db9de01e00f37a7a0 2024-11-19T11:16:14,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/8612cefb3b6844a59872760a670ec125 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/8612cefb3b6844a59872760a670ec125 2024-11-19T11:16:14,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/8612cefb3b6844a59872760a670ec125, entries=150, sequenceid=416, filesize=30.5 K 2024-11-19T11:16:14,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/320846b6d4724ba9aa7552a7911e8acb as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/320846b6d4724ba9aa7552a7911e8acb 2024-11-19T11:16:14,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/320846b6d4724ba9aa7552a7911e8acb, entries=150, sequenceid=416, filesize=12.0 K 2024-11-19T11:16:14,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/8cc2eafdea724f3db9de01e00f37a7a0 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8cc2eafdea724f3db9de01e00f37a7a0 2024-11-19T11:16:14,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8cc2eafdea724f3db9de01e00f37a7a0, entries=150, sequenceid=416, filesize=12.0 K 2024-11-19T11:16:14,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=20.13 KB/20610 for d631ed8d9a4ea7a066117289c5e10132 in 1684ms, sequenceid=416, compaction requested=true 2024-11-19T11:16:14,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d631ed8d9a4ea7a066117289c5e10132: 2024-11-19T11:16:14,904 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:14,904 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:14,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d631ed8d9a4ea7a066117289c5e10132:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:16:14,904 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. after waiting 0 ms 2024-11-19T11:16:14,904 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. because compaction request was cancelled 2024-11-19T11:16:14,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:14,904 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:14,904 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d631ed8d9a4ea7a066117289c5e10132:A 2024-11-19T11:16:14,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d631ed8d9a4ea7a066117289c5e10132:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:16:14,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:14,904 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. because compaction request was cancelled 2024-11-19T11:16:14,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d631ed8d9a4ea7a066117289c5e10132:C, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:16:14,904 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing d631ed8d9a4ea7a066117289c5e10132 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-19T11:16:14,904 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d631ed8d9a4ea7a066117289c5e10132:B 2024-11-19T11:16:14,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:14,904 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. because compaction request was cancelled 2024-11-19T11:16:14,904 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d631ed8d9a4ea7a066117289c5e10132:C 2024-11-19T11:16:14,904 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=A 2024-11-19T11:16:14,904 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:14,904 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=B 2024-11-19T11:16:14,904 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:14,904 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d631ed8d9a4ea7a066117289c5e10132, store=C 2024-11-19T11:16:14,904 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:14,911 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191ac638ed37f44f92914b7cc4e0cfcff4_d631ed8d9a4ea7a066117289c5e10132 is 50, key is test_row_0/A:col10/1732014973498/Put/seqid=0 2024-11-19T11:16:14,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742261_1437 (size=12454) 2024-11-19T11:16:15,316 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:15,322 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191ac638ed37f44f92914b7cc4e0cfcff4_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191ac638ed37f44f92914b7cc4e0cfcff4_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:15,323 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/0ac85a7359e44f55a022799ed67ca32f, store: [table=TestAcidGuarantees family=A region=d631ed8d9a4ea7a066117289c5e10132] 2024-11-19T11:16:15,324 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/0ac85a7359e44f55a022799ed67ca32f is 175, key is test_row_0/A:col10/1732014973498/Put/seqid=0 2024-11-19T11:16:15,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742262_1438 (size=31255) 2024-11-19T11:16:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T11:16:15,729 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=422, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/0ac85a7359e44f55a022799ed67ca32f 2024-11-19T11:16:15,734 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/ce781ce5f1764ba6b47089fec739ca01 is 50, key is test_row_0/B:col10/1732014973498/Put/seqid=0 2024-11-19T11:16:15,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742263_1439 (size=12301) 2024-11-19T11:16:16,138 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/ce781ce5f1764ba6b47089fec739ca01 2024-11-19T11:16:16,143 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/cdd9cbf7c31a444999fd586d1bfd6ffe is 50, key is test_row_0/C:col10/1732014973498/Put/seqid=0 2024-11-19T11:16:16,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742264_1440 (size=12301) 2024-11-19T11:16:16,547 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/cdd9cbf7c31a444999fd586d1bfd6ffe 2024-11-19T11:16:16,553 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/A/0ac85a7359e44f55a022799ed67ca32f as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/0ac85a7359e44f55a022799ed67ca32f 2024-11-19T11:16:16,558 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/0ac85a7359e44f55a022799ed67ca32f, entries=150, sequenceid=422, filesize=30.5 K 2024-11-19T11:16:16,559 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/B/ce781ce5f1764ba6b47089fec739ca01 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/ce781ce5f1764ba6b47089fec739ca01 2024-11-19T11:16:16,563 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/ce781ce5f1764ba6b47089fec739ca01, entries=150, sequenceid=422, filesize=12.0 K 2024-11-19T11:16:16,564 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/.tmp/C/cdd9cbf7c31a444999fd586d1bfd6ffe as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/cdd9cbf7c31a444999fd586d1bfd6ffe 2024-11-19T11:16:16,567 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/cdd9cbf7c31a444999fd586d1bfd6ffe, entries=150, sequenceid=422, filesize=12.0 K 2024-11-19T11:16:16,568 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for d631ed8d9a4ea7a066117289c5e10132 in 1664ms, sequenceid=422, compaction requested=true 2024-11-19T11:16:16,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/bc1c6e3291e44f509a3462b45e665f8b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/1216e1a3ce0f4e83adb390e3e188b4a4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/f59794ad2d2b4c2ba1c6bc5d908abb69, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/f98ca166e12048efb1d209bbdb071bc5, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/c8e18728acb44f42901d881a6180220b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/b4567bb83a7640c68bf3bf550ce32ed2, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/23f42e682e144cb58a6d7e8500637630, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/9db11ed13f2f4d748d27ac4292f5bc8b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/deeae19b57ae44cd97003123c1213d72, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/b33883e363664d4c91f48f30545be293, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/f9d0ace55dd540849de76728798fe2b1, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/7d505b1e177943c7b808a6b76d66f0d9, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/74748d08055c440fb7af8f42da0eda34, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/b4f86a7c7f44448798c5cb75bc0d794c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/287dc1d624d24625aa492d35bd96f350, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/27b59f3f15e94c10918e4590afebc06d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/d8b12103057a44e1910948d3bcd56f5c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/4ec2bf5b61bb488b87b362dd834c7dbb, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/d83e75c2f6714cf9b8dc614d02a36727] to archive 2024-11-19T11:16:16,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T11:16:16,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/bc1c6e3291e44f509a3462b45e665f8b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/bc1c6e3291e44f509a3462b45e665f8b 2024-11-19T11:16:16,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/1216e1a3ce0f4e83adb390e3e188b4a4 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/1216e1a3ce0f4e83adb390e3e188b4a4 2024-11-19T11:16:16,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/f59794ad2d2b4c2ba1c6bc5d908abb69 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/f59794ad2d2b4c2ba1c6bc5d908abb69 2024-11-19T11:16:16,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/f98ca166e12048efb1d209bbdb071bc5 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/f98ca166e12048efb1d209bbdb071bc5 2024-11-19T11:16:16,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/c8e18728acb44f42901d881a6180220b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/c8e18728acb44f42901d881a6180220b 2024-11-19T11:16:16,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/b4567bb83a7640c68bf3bf550ce32ed2 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/b4567bb83a7640c68bf3bf550ce32ed2 2024-11-19T11:16:16,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/23f42e682e144cb58a6d7e8500637630 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/23f42e682e144cb58a6d7e8500637630 2024-11-19T11:16:16,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/9db11ed13f2f4d748d27ac4292f5bc8b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/9db11ed13f2f4d748d27ac4292f5bc8b 2024-11-19T11:16:16,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/deeae19b57ae44cd97003123c1213d72 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/deeae19b57ae44cd97003123c1213d72 2024-11-19T11:16:16,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/b33883e363664d4c91f48f30545be293 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/b33883e363664d4c91f48f30545be293 2024-11-19T11:16:16,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/f9d0ace55dd540849de76728798fe2b1 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/f9d0ace55dd540849de76728798fe2b1 2024-11-19T11:16:16,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/7d505b1e177943c7b808a6b76d66f0d9 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/7d505b1e177943c7b808a6b76d66f0d9 2024-11-19T11:16:16,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/74748d08055c440fb7af8f42da0eda34 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/74748d08055c440fb7af8f42da0eda34 2024-11-19T11:16:16,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/b4f86a7c7f44448798c5cb75bc0d794c to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/b4f86a7c7f44448798c5cb75bc0d794c 2024-11-19T11:16:16,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/287dc1d624d24625aa492d35bd96f350 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/287dc1d624d24625aa492d35bd96f350 2024-11-19T11:16:16,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/27b59f3f15e94c10918e4590afebc06d to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/27b59f3f15e94c10918e4590afebc06d 2024-11-19T11:16:16,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/d8b12103057a44e1910948d3bcd56f5c to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/d8b12103057a44e1910948d3bcd56f5c 2024-11-19T11:16:16,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/4ec2bf5b61bb488b87b362dd834c7dbb to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/4ec2bf5b61bb488b87b362dd834c7dbb 2024-11-19T11:16:16,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/d83e75c2f6714cf9b8dc614d02a36727 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/d83e75c2f6714cf9b8dc614d02a36727 2024-11-19T11:16:16,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/d8af6d65385f48ba8cf513b7fd14eed3, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/1aee25267bc544179fb950c634534964, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/bc4ea39c11ea40089b689f07c29dfc52, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/045bfce002a546bfb189825be44ed65a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/185e9d03282d461c95cadad500dbeafc, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/be49d5608f5141e1875e91d308f5f2ee, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/32acf037b7644aefbb921c38dc31a9e6, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/47c1aa8328774cc9b444a55b260ac65a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/fec6bde1efe545c78afa82ce91a88b28, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/ed39917aac88405e838612acd6aef325, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/41c5ac048b2647bf98b540e81ef29c44, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/93898131748343ba84b89dfb48a671b7, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/103ff3de3df14512bff746253ff91308, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/3dbba35b5c8d4955bd50b7682ec224d7, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/2b20a60bf6fb493485fbffdfd848aca6, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/0b02d7cbcb2347a3bdca3d4f0daab65b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/9761fddb57ab42e99f66bfab8bed597b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/6126ca3080744eb09b0261c973fcab69, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/8a872aa916d948c6aceb44ff198a517c] to archive 2024-11-19T11:16:16,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T11:16:16,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/d8af6d65385f48ba8cf513b7fd14eed3 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/d8af6d65385f48ba8cf513b7fd14eed3 2024-11-19T11:16:16,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/1aee25267bc544179fb950c634534964 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/1aee25267bc544179fb950c634534964 2024-11-19T11:16:16,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/bc4ea39c11ea40089b689f07c29dfc52 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/bc4ea39c11ea40089b689f07c29dfc52 2024-11-19T11:16:16,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/045bfce002a546bfb189825be44ed65a to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/045bfce002a546bfb189825be44ed65a 2024-11-19T11:16:16,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/185e9d03282d461c95cadad500dbeafc to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/185e9d03282d461c95cadad500dbeafc 2024-11-19T11:16:16,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/be49d5608f5141e1875e91d308f5f2ee to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/be49d5608f5141e1875e91d308f5f2ee 2024-11-19T11:16:16,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/32acf037b7644aefbb921c38dc31a9e6 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/32acf037b7644aefbb921c38dc31a9e6 2024-11-19T11:16:16,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/47c1aa8328774cc9b444a55b260ac65a to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/47c1aa8328774cc9b444a55b260ac65a 2024-11-19T11:16:16,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/fec6bde1efe545c78afa82ce91a88b28 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/fec6bde1efe545c78afa82ce91a88b28 2024-11-19T11:16:16,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/ed39917aac88405e838612acd6aef325 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/ed39917aac88405e838612acd6aef325 2024-11-19T11:16:16,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/41c5ac048b2647bf98b540e81ef29c44 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/41c5ac048b2647bf98b540e81ef29c44 2024-11-19T11:16:16,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/93898131748343ba84b89dfb48a671b7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/93898131748343ba84b89dfb48a671b7 2024-11-19T11:16:16,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/103ff3de3df14512bff746253ff91308 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/103ff3de3df14512bff746253ff91308 2024-11-19T11:16:16,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/3dbba35b5c8d4955bd50b7682ec224d7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/3dbba35b5c8d4955bd50b7682ec224d7 2024-11-19T11:16:16,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/2b20a60bf6fb493485fbffdfd848aca6 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/2b20a60bf6fb493485fbffdfd848aca6 2024-11-19T11:16:16,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/0b02d7cbcb2347a3bdca3d4f0daab65b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/0b02d7cbcb2347a3bdca3d4f0daab65b 2024-11-19T11:16:16,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/9761fddb57ab42e99f66bfab8bed597b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/9761fddb57ab42e99f66bfab8bed597b 2024-11-19T11:16:16,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/6126ca3080744eb09b0261c973fcab69 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/6126ca3080744eb09b0261c973fcab69 2024-11-19T11:16:16,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/8a872aa916d948c6aceb44ff198a517c to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/8a872aa916d948c6aceb44ff198a517c 2024-11-19T11:16:16,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8af1e6b1b2684c6781b67b3ea6562986, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/082e9053e3884e129c23de59689ef0a2, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8fa44d0b59744a158b6c75ed217fa706, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/894c5d8ff650451785c7b24089cbe598, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/d7cafa7074c94e128676a8d63bd7353a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/599654f729614e758bfc36e2e7c3e55d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/7a000616336d4940833b58397d40429d, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/c30b92b28d114cd3adc3fd648a7abb77, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/f3649235dcfb4d21b0cc2f36a56b2ba7, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/fcc8afb1852f4bafac7a0269c7389feb, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/bf575832dd77437eb35a9edccfaff989, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8ae7bc63c6e046aa9fd5b673defa5542, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/0f74082d04e64f4ebb47f2a4ea77c222, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/42fb6c75891443eda454b118e269f5f4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/4dbe9f64eb1047c0ab95ecb3e2f9b0b8, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/35333dd8c46440d780894eb0af3d4041, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/36a95b41dc8248c2b5af40e33e54e625, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/50a762c0cd9f478f942a8610d4927c49, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/5a909447dee348f4b11f640c0a24c6f0] to archive 2024-11-19T11:16:16,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T11:16:16,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8af1e6b1b2684c6781b67b3ea6562986 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8af1e6b1b2684c6781b67b3ea6562986 2024-11-19T11:16:16,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/082e9053e3884e129c23de59689ef0a2 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/082e9053e3884e129c23de59689ef0a2 2024-11-19T11:16:16,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8fa44d0b59744a158b6c75ed217fa706 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8fa44d0b59744a158b6c75ed217fa706 2024-11-19T11:16:16,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/894c5d8ff650451785c7b24089cbe598 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/894c5d8ff650451785c7b24089cbe598 2024-11-19T11:16:16,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/d7cafa7074c94e128676a8d63bd7353a to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/d7cafa7074c94e128676a8d63bd7353a 2024-11-19T11:16:16,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/599654f729614e758bfc36e2e7c3e55d to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/599654f729614e758bfc36e2e7c3e55d 2024-11-19T11:16:16,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/7a000616336d4940833b58397d40429d to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/7a000616336d4940833b58397d40429d 2024-11-19T11:16:16,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/c30b92b28d114cd3adc3fd648a7abb77 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/c30b92b28d114cd3adc3fd648a7abb77 2024-11-19T11:16:16,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/f3649235dcfb4d21b0cc2f36a56b2ba7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/f3649235dcfb4d21b0cc2f36a56b2ba7 2024-11-19T11:16:16,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/fcc8afb1852f4bafac7a0269c7389feb to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/fcc8afb1852f4bafac7a0269c7389feb 2024-11-19T11:16:16,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/bf575832dd77437eb35a9edccfaff989 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/bf575832dd77437eb35a9edccfaff989 2024-11-19T11:16:16,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8ae7bc63c6e046aa9fd5b673defa5542 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8ae7bc63c6e046aa9fd5b673defa5542 2024-11-19T11:16:16,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/0f74082d04e64f4ebb47f2a4ea77c222 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/0f74082d04e64f4ebb47f2a4ea77c222 2024-11-19T11:16:16,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/42fb6c75891443eda454b118e269f5f4 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/42fb6c75891443eda454b118e269f5f4 2024-11-19T11:16:16,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/4dbe9f64eb1047c0ab95ecb3e2f9b0b8 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/4dbe9f64eb1047c0ab95ecb3e2f9b0b8 2024-11-19T11:16:16,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/35333dd8c46440d780894eb0af3d4041 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/35333dd8c46440d780894eb0af3d4041 2024-11-19T11:16:16,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/36a95b41dc8248c2b5af40e33e54e625 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/36a95b41dc8248c2b5af40e33e54e625 2024-11-19T11:16:16,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/50a762c0cd9f478f942a8610d4927c49 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/50a762c0cd9f478f942a8610d4927c49 2024-11-19T11:16:16,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/5a909447dee348f4b11f640c0a24c6f0 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/5a909447dee348f4b11f640c0a24c6f0 2024-11-19T11:16:16,662 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/recovered.edits/425.seqid, newMaxSeqId=425, maxSeqId=4 2024-11-19T11:16:16,662 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. 2024-11-19T11:16:16,662 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for d631ed8d9a4ea7a066117289c5e10132: Waiting for close lock at 1732014973667Running coprocessor pre-close hooks at 1732014973667Disabling compacts and flushes for region at 1732014973667Disabling writes for close at 1732014974904 (+1237 ms)Obtaining lock to block concurrent updates at 1732014974904Preparing flush snapshotting stores in d631ed8d9a4ea7a066117289c5e10132 at 1732014974904Finished memstore snapshotting TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732014974904Flushing stores of TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132. at 1732014974905 (+1 ms)Flushing d631ed8d9a4ea7a066117289c5e10132/A: creating writer at 1732014974905Flushing mob file d631ed8d9a4ea7a066117289c5e10132/A: appending metadata at 1732014974911 (+6 ms)Flushing mob file d631ed8d9a4ea7a066117289c5e10132/A: closing flushed file at 1732014974911Flushing d631ed8d9a4ea7a066117289c5e10132/A: appending metadata at 1732014975323 (+412 ms)Flushing d631ed8d9a4ea7a066117289c5e10132/A: closing flushed file at 1732014975324 (+1 ms)Flushing d631ed8d9a4ea7a066117289c5e10132/B: creating writer at 1732014975732 (+408 ms)Flushing d631ed8d9a4ea7a066117289c5e10132/B: appending metadata at 1732014975734 (+2 ms)Flushing d631ed8d9a4ea7a066117289c5e10132/B: closing flushed file at 1732014975734Flushing d631ed8d9a4ea7a066117289c5e10132/C: creating writer at 1732014976141 (+407 ms)Flushing d631ed8d9a4ea7a066117289c5e10132/C: appending metadata at 1732014976143 (+2 ms)Flushing d631ed8d9a4ea7a066117289c5e10132/C: closing flushed file at 1732014976143Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@179771ea: reopening flushed file at 1732014976552 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7083644e: reopening flushed file at 1732014976558 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e4a6ffe: reopening flushed file at 1732014976563 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for d631ed8d9a4ea7a066117289c5e10132 in 1664ms, sequenceid=422, compaction requested=true at 1732014976568 (+5 ms)Writing region close event to WAL at 1732014976659 (+91 ms)Running coprocessor post-close hooks at 1732014976662 (+3 ms)Closed at 1732014976662 2024-11-19T11:16:16,664 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:16,665 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=d631ed8d9a4ea7a066117289c5e10132, regionState=CLOSED 2024-11-19T11:16:16,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 because future has completed 2024-11-19T11:16:16,669 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-19T11:16:16,669 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure d631ed8d9a4ea7a066117289c5e10132, server=16c88966c0af,35833,1732014855640 in 3.1520 sec 2024-11-19T11:16:16,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-19T11:16:16,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d631ed8d9a4ea7a066117289c5e10132, UNASSIGN in 3.1570 sec 2024-11-19T11:16:16,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-19T11:16:16,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.1590 sec 2024-11-19T11:16:16,673 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014976673"}]},"ts":"1732014976673"} 2024-11-19T11:16:16,674 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T11:16:16,674 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T11:16:16,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.1690 sec 2024-11-19T11:16:17,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T11:16:17,669 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:16:17,670 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T11:16:17,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:16:17,674 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:16:17,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T11:16:17,675 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:16:17,679 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:17,682 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A, FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B, FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C, FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/recovered.edits] 2024-11-19T11:16:17,685 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/0ac85a7359e44f55a022799ed67ca32f to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/0ac85a7359e44f55a022799ed67ca32f 2024-11-19T11:16:17,686 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/17952deb21a34760906fcfe40f0e4d83 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/17952deb21a34760906fcfe40f0e4d83 2024-11-19T11:16:17,688 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/391175da6d694506b2dc8f4f120c877c to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/391175da6d694506b2dc8f4f120c877c 2024-11-19T11:16:17,689 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/3b1dd4a4284a4f1a8459d0cbcaf090e4 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/3b1dd4a4284a4f1a8459d0cbcaf090e4 2024-11-19T11:16:17,690 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/8612cefb3b6844a59872760a670ec125 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/A/8612cefb3b6844a59872760a670ec125 2024-11-19T11:16:17,692 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/06e7a207f911499bb9f96d3d2e05b0cd to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/06e7a207f911499bb9f96d3d2e05b0cd 2024-11-19T11:16:17,692 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/320846b6d4724ba9aa7552a7911e8acb to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/320846b6d4724ba9aa7552a7911e8acb 2024-11-19T11:16:17,693 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/c0722578b00d4ea1bf301580975ea6d7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/c0722578b00d4ea1bf301580975ea6d7 2024-11-19T11:16:17,694 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/ce781ce5f1764ba6b47089fec739ca01 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/ce781ce5f1764ba6b47089fec739ca01 2024-11-19T11:16:17,695 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/f0116ce5607d4b18a2e0cbe630b9e1d4 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/B/f0116ce5607d4b18a2e0cbe630b9e1d4 2024-11-19T11:16:17,697 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/256934d1783144b4a9f9a7ada8b74f64 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/256934d1783144b4a9f9a7ada8b74f64 2024-11-19T11:16:17,697 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/697766f3ac7d4613b577c8a854f4f618 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/697766f3ac7d4613b577c8a854f4f618 2024-11-19T11:16:17,698 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8cc2eafdea724f3db9de01e00f37a7a0 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8cc2eafdea724f3db9de01e00f37a7a0 2024-11-19T11:16:17,699 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8ce4183aabfe4afbac6a38cb1fea5065 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/8ce4183aabfe4afbac6a38cb1fea5065 2024-11-19T11:16:17,699 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/cdd9cbf7c31a444999fd586d1bfd6ffe to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/C/cdd9cbf7c31a444999fd586d1bfd6ffe 2024-11-19T11:16:17,701 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/recovered.edits/425.seqid to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132/recovered.edits/425.seqid 2024-11-19T11:16:17,702 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:17,702 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T11:16:17,702 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T11:16:17,702 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T11:16:17,704 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111919d1b0058885402fb6290a82bb98f036_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111919d1b0058885402fb6290a82bb98f036_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:17,705 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191ac638ed37f44f92914b7cc4e0cfcff4_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191ac638ed37f44f92914b7cc4e0cfcff4_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:17,706 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111962c30aab9eec41f7abc844db84484193_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111962c30aab9eec41f7abc844db84484193_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:17,707 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111967157c75770a48a7a20d395f489fbbc2_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111967157c75770a48a7a20d395f489fbbc2_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:17,708 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111979e3f5e7aa8c4319803072ddc732eaed_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111979e3f5e7aa8c4319803072ddc732eaed_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:17,708 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119857066ac9d724aeaa8f3d6c5055f2a9a_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119857066ac9d724aeaa8f3d6c5055f2a9a_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:17,709 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199b2a882803e74ebfa45aec81fafb4e58_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199b2a882803e74ebfa45aec81fafb4e58_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:17,710 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199b6ff2af9ce14c958b6e6f2d583a85be_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199b6ff2af9ce14c958b6e6f2d583a85be_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:17,710 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a1ac451156a64430927ed3c71ff535e4_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a1ac451156a64430927ed3c71ff535e4_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:17,711 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b7e40c98d29e46a8b3571f11c861ed39_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b7e40c98d29e46a8b3571f11c861ed39_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:17,712 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bb7134ef809142e8baf23f48e0aa2ee5_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bb7134ef809142e8baf23f48e0aa2ee5_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:17,713 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bc7e028fb80e4693a4b646e93e888cd0_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bc7e028fb80e4693a4b646e93e888cd0_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:17,714 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c24ee4bb97bf4aa197d1dd94b83e9cc6_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c24ee4bb97bf4aa197d1dd94b83e9cc6_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:17,714 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119cb241b64e2fe4f0ba181b2f541061722_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119cb241b64e2fe4f0ba181b2f541061722_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:17,715 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d00c853bdf714f8ca9a22545afb9d21c_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d00c853bdf714f8ca9a22545afb9d21c_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:17,716 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e34497f302ed4ffa986b41137e9a4c8c_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e34497f302ed4ffa986b41137e9a4c8c_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:17,717 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ea506d1e8c7a48848ad4bc5d9a23149d_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ea506d1e8c7a48848ad4bc5d9a23149d_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:17,717 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f7ae1a6a184b437a899a0354903a0e23_d631ed8d9a4ea7a066117289c5e10132 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f7ae1a6a184b437a899a0354903a0e23_d631ed8d9a4ea7a066117289c5e10132 2024-11-19T11:16:17,718 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T11:16:17,719 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:16:17,722 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T11:16:17,724 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T11:16:17,725 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:16:17,725 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T11:16:17,725 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732014977725"}]},"ts":"9223372036854775807"} 2024-11-19T11:16:17,727 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T11:16:17,727 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d631ed8d9a4ea7a066117289c5e10132, NAME => 'TestAcidGuarantees,,1732014949410.d631ed8d9a4ea7a066117289c5e10132.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T11:16:17,727 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T11:16:17,727 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732014977727"}]},"ts":"9223372036854775807"} 2024-11-19T11:16:17,728 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T11:16:17,729 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:16:17,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 59 msec 2024-11-19T11:16:17,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T11:16:17,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T11:16:17,778 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:16:17,787 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 246) - Thread LEAK? -, OpenFileDescriptor=461 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=414 (was 418), ProcessCount=11 (was 11), AvailableMemoryMB=11682 (was 12243) 2024-11-19T11:16:17,795 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=414, ProcessCount=11, AvailableMemoryMB=11682 2024-11-19T11:16:17,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T11:16:17,796 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:16:17,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T11:16:17,798 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T11:16:17,798 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:17,798 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-11-19T11:16:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-19T11:16:17,799 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T11:16:17,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742265_1441 (size=1061) 2024-11-19T11:16:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-19T11:16:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-19T11:16:18,206 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094 2024-11-19T11:16:18,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742266_1442 (size=53) 2024-11-19T11:16:18,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-19T11:16:18,615 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:16:18,616 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5fb872772cd7220dc7447361f8e609a9, disabling compactions & flushes 2024-11-19T11:16:18,616 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:18,616 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:18,616 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. after waiting 0 ms 2024-11-19T11:16:18,616 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:18,616 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:18,616 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5fb872772cd7220dc7447361f8e609a9: Waiting for close lock at 1732014978616Disabling compacts and flushes for region at 1732014978616Disabling writes for close at 1732014978616Writing region close event to WAL at 1732014978616Closed at 1732014978616 2024-11-19T11:16:18,617 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T11:16:18,618 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732014978617"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732014978617"}]},"ts":"1732014978617"} 2024-11-19T11:16:18,619 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T11:16:18,621 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T11:16:18,621 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014978621"}]},"ts":"1732014978621"} 2024-11-19T11:16:18,624 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T11:16:18,624 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5fb872772cd7220dc7447361f8e609a9, ASSIGN}] 2024-11-19T11:16:18,626 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5fb872772cd7220dc7447361f8e609a9, ASSIGN 2024-11-19T11:16:18,627 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5fb872772cd7220dc7447361f8e609a9, ASSIGN; state=OFFLINE, location=16c88966c0af,35833,1732014855640; forceNewPlan=false, retain=false 2024-11-19T11:16:18,778 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=5fb872772cd7220dc7447361f8e609a9, regionState=OPENING, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:16:18,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5fb872772cd7220dc7447361f8e609a9, ASSIGN because future has completed 2024-11-19T11:16:18,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640}] 2024-11-19T11:16:18,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-19T11:16:18,939 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:18,939 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} 2024-11-19T11:16:18,941 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:18,941 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:16:18,941 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:18,941 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:18,943 INFO [StoreOpener-5fb872772cd7220dc7447361f8e609a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:18,945 INFO [StoreOpener-5fb872772cd7220dc7447361f8e609a9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:16:18,945 INFO [StoreOpener-5fb872772cd7220dc7447361f8e609a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5fb872772cd7220dc7447361f8e609a9 columnFamilyName A 2024-11-19T11:16:18,945 DEBUG [StoreOpener-5fb872772cd7220dc7447361f8e609a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:18,946 INFO [StoreOpener-5fb872772cd7220dc7447361f8e609a9-1 {}] regionserver.HStore(327): Store=5fb872772cd7220dc7447361f8e609a9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:16:18,946 INFO [StoreOpener-5fb872772cd7220dc7447361f8e609a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:18,947 INFO [StoreOpener-5fb872772cd7220dc7447361f8e609a9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:16:18,947 INFO [StoreOpener-5fb872772cd7220dc7447361f8e609a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5fb872772cd7220dc7447361f8e609a9 columnFamilyName B 2024-11-19T11:16:18,947 DEBUG [StoreOpener-5fb872772cd7220dc7447361f8e609a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:18,948 INFO [StoreOpener-5fb872772cd7220dc7447361f8e609a9-1 {}] regionserver.HStore(327): Store=5fb872772cd7220dc7447361f8e609a9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:16:18,948 INFO [StoreOpener-5fb872772cd7220dc7447361f8e609a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:18,949 INFO [StoreOpener-5fb872772cd7220dc7447361f8e609a9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:16:18,949 INFO [StoreOpener-5fb872772cd7220dc7447361f8e609a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5fb872772cd7220dc7447361f8e609a9 columnFamilyName C 2024-11-19T11:16:18,949 DEBUG [StoreOpener-5fb872772cd7220dc7447361f8e609a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:18,950 INFO [StoreOpener-5fb872772cd7220dc7447361f8e609a9-1 {}] regionserver.HStore(327): Store=5fb872772cd7220dc7447361f8e609a9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:16:18,950 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:18,950 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:18,951 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:18,951 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:18,952 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:18,952 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:18,953 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T11:16:18,954 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:18,955 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T11:16:18,956 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened 5fb872772cd7220dc7447361f8e609a9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67051927, jitterRate=-8.48427414894104E-4}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T11:16:18,956 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:18,956 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for 5fb872772cd7220dc7447361f8e609a9: Running coprocessor pre-open hook at 1732014978942Writing region info on filesystem at 1732014978942Initializing all the Stores at 1732014978943 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014978943Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014978943Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014978943Cleaning up temporary data from old regions at 1732014978952 (+9 ms)Running coprocessor post-open hooks at 1732014978956 (+4 ms)Region opened successfully at 1732014978956 2024-11-19T11:16:18,957 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., pid=120, masterSystemTime=1732014978934 2024-11-19T11:16:18,959 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:18,959 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:18,959 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=5fb872772cd7220dc7447361f8e609a9, regionState=OPEN, openSeqNum=2, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:16:18,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 because future has completed 2024-11-19T11:16:18,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-19T11:16:18,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure 5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 in 180 msec 2024-11-19T11:16:18,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-19T11:16:18,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5fb872772cd7220dc7447361f8e609a9, ASSIGN in 339 msec 2024-11-19T11:16:18,966 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T11:16:18,966 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014978966"}]},"ts":"1732014978966"} 2024-11-19T11:16:18,968 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T11:16:18,969 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T11:16:18,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1730 sec 2024-11-19T11:16:19,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-19T11:16:19,947 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:16:19,948 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ad78a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,949 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:16:19,949 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:16:19,950 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:16:19,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:16:19,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:16:19,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@756d5016, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:16:19,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:16:19,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:19,951 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:16:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@212347ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,952 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:16:19,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:16:19,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:16:19,953 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:16:19,954 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T11:16:19,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 16c88966c0af,41441,1732014854424 2024-11-19T11:16:19,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4df2c421 2024-11-19T11:16:19,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T11:16:19,955 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T11:16:19,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,956 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:16:19,956 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:16:19,957 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:16:19,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:16:19,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:16:19,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5923d651, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:16:19,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:16:19,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:19,958 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:16:19,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3209cdd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,959 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:16:19,959 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:16:19,960 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:16:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:16:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:16:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ae548d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:16:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:16:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:19,961 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:16:19,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@532f164c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,962 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,962 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:16:19,962 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:16:19,962 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:16:19,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:16:19,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:16:19,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67d116f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:16:19,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:16:19,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:19,963 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:16:19,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@324b4cb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,964 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,964 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:16:19,964 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:16:19,965 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:16:19,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:16:19,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:16:19,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f8a68cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:16:19,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:16:19,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:19,966 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:16:19,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ea7bbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,967 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:16:19,967 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:16:19,968 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:16:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:16:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:16:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@512f2c74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:16:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:16:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:19,969 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:16:19,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2415d7dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,970 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:16:19,970 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:16:19,971 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:16:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:16:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:16:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21dfef85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:16:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:16:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:19,972 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:16:19,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4177ce82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,973 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,973 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:16:19,973 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:16:19,974 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:16:19,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:16:19,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:16:19,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dbf7439, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:16:19,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:16:19,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:19,976 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:16:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c1cff17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:16:19,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:16:19,978 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:16:19,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:16:19,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:16:19,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20822bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:16:19,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:16:19,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:19,979 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:16:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56e8b160, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:16:19,980 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:16:19,981 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:16:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:16:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:16:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ce93a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:16:19,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:16:19,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:19,982 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:16:19,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3384b4d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df5a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,984 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:16:19,984 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:16:19,984 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:16:19,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:16:19,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:16:19,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19496eff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:16:19,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:16:19,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:19,986 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:16:19,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb60c02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:19,987 DEBUG [Thread-1989 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:16:19,988 DEBUG [Thread-2000 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:16:19,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:16:19,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:16:19,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:16:19,988 DEBUG [Thread-1993 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:16:19,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:16:19,989 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:16:19,989 DEBUG [Thread-2004 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:16:19,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:16:19,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:16:19,989 DEBUG [Thread-2002 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:16:19,989 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:16:19,990 DEBUG [Thread-2006 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:16:19,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:16:19,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:16:19,990 DEBUG [Thread-1995 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:16:19,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:16:19,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:16:19,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:16:19,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:16:19,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:16:19,992 DEBUG [Thread-1987 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:16:19,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:16:19,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:16:19,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:16:19,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:16:19,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:16:19,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:16:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:16:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:16:19,994 DEBUG [Thread-1991 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:16:19,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:16:19,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:16:19,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:16:19,994 DEBUG [Thread-1998 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:16:19,995 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:16:19,995 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:16:19,995 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:16:19,996 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:16:19,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:16:19,996 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:16:19,996 DEBUG [Thread-1991 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:16:19,997 DEBUG [Thread-1991 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:16:19,997 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:16:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:16:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:16:19,998 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:16:19,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-19T11:16:19,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:16:20,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T11:16:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:16:20,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2] 2024-11-19T11:16:20,001 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:16:20,002 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:16:20,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:16:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:20,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5fb872772cd7220dc7447361f8e609a9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T11:16:20,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=A 2024-11-19T11:16:20,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:20,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=B 2024-11-19T11:16:20,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:20,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=C 2024-11-19T11:16:20,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:20,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/a67d2c6cad094fb5b0d0a00ca394c147 is 50, key is test_row_0/A:col10/1732014980006/Put/seqid=0 2024-11-19T11:16:20,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742267_1443 (size=12001) 2024-11-19T11:16:20,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015040035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:20,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015040036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:20,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015040037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:20,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015040040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:20,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014980034, max timestamp=1732014980045Num uniques 859; 2024-11-19T11:16:20,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014980034, max timestamp=1732014980045Num uniques 859; 2024-11-19T11:16:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:20,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014980034, max timestamp=1732014980045Num uniques 859; 2024-11-19T11:16:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:20,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T11:16:20,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:20,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015040129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:20,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:20,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T11:16:20,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:20,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:20,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:20,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:20,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:20,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:20,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015040158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:20,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:20,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015040158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:20,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015040158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:20,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:20,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015040158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:20,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:20,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015040237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:20,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:20,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T11:16:20,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:20,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:20,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:20,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:20,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:20,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T11:16:20,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015040367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015040367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:20,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015040367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:20,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:20,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015040377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:20,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/a67d2c6cad094fb5b0d0a00ca394c147 2024-11-19T11:16:20,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015040447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:20,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:20,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T11:16:20,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:20,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:20,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:20,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:20,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:20,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/bd8d18e33ba040d9ac13d31d7a4b10cb is 50, key is test_row_0/B:col10/1732014980006/Put/seqid=0 2024-11-19T11:16:20,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742268_1444 (size=12001) 2024-11-19T11:16:20,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T11:16:20,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:20,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:20,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:20,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:20,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:20,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:20,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T11:16:20,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:20,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015040677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:20,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:20,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015040677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:20,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:20,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:20,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:20,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015040687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:20,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:20,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:20,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015040698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:20,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:20,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015040757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:20,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:20,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:20,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T11:16:20,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:20,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:20,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:20,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:20,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:20,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:20,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/bd8d18e33ba040d9ac13d31d7a4b10cb 2024-11-19T11:16:20,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/5084288f6735469d9678a3830c8e68e4 is 50, key is test_row_0/C:col10/1732014980006/Put/seqid=0 2024-11-19T11:16:20,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T11:16:20,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:20,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:20,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:20,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:20,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:20,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742269_1445 (size=12001) 2024-11-19T11:16:21,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T11:16:21,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:21,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:21,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:21,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:21,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:21,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T11:16:21,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:21,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015041197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:21,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:21,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:21,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:21,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015041197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:21,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:21,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:21,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:21,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:21,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:21,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015041207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:21,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:21,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:21,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:21,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:21,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015041218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:21,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:21,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:21,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:21,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T11:16:21,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:21,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:21,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:21,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:21,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:21,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:21,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:21,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015041277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:21,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:21,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:21,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:21,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/5084288f6735469d9678a3830c8e68e4 2024-11-19T11:16:21,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/a67d2c6cad094fb5b0d0a00ca394c147 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/a67d2c6cad094fb5b0d0a00ca394c147 2024-11-19T11:16:21,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/a67d2c6cad094fb5b0d0a00ca394c147, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T11:16:21,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/bd8d18e33ba040d9ac13d31d7a4b10cb as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/bd8d18e33ba040d9ac13d31d7a4b10cb 2024-11-19T11:16:21,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/bd8d18e33ba040d9ac13d31d7a4b10cb, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T11:16:21,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/5084288f6735469d9678a3830c8e68e4 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/5084288f6735469d9678a3830c8e68e4 2024-11-19T11:16:21,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/5084288f6735469d9678a3830c8e68e4, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T11:16:21,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 5fb872772cd7220dc7447361f8e609a9 in 1352ms, sequenceid=12, compaction requested=false 2024-11-19T11:16:21,358 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T11:16:21,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:21,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T11:16:21,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:21,384 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 5fb872772cd7220dc7447361f8e609a9 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-19T11:16:21,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=A 2024-11-19T11:16:21,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:21,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=B 2024-11-19T11:16:21,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:21,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=C 2024-11-19T11:16:21,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:21,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/aa1b63e2c48a41d58652b8d79411fd67 is 50, key is test_row_0/A:col10/1732014980098/Put/seqid=0 2024-11-19T11:16:21,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742270_1446 (size=12001) 2024-11-19T11:16:21,791 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/aa1b63e2c48a41d58652b8d79411fd67 2024-11-19T11:16:21,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/d5bf016fa74e4562a118442aedcf00fa is 50, key is test_row_0/B:col10/1732014980098/Put/seqid=0 2024-11-19T11:16:21,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742271_1447 (size=12001) 2024-11-19T11:16:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T11:16:22,185 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T11:16:22,211 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/d5bf016fa74e4562a118442aedcf00fa 2024-11-19T11:16:22,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:22,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/268b721b842649fba22d492ab1f37e59 is 50, key is test_row_0/C:col10/1732014980098/Put/seqid=0 2024-11-19T11:16:22,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742272_1448 (size=12001) 2024-11-19T11:16:22,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015042263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:22,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015042265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:22,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015042266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015042267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:22,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015042297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:22,304 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_1' in 5fb872772cd7220dc7447361f8e609a9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:22,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015042377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:22,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:22,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015042387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:22,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:22,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015042387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:22,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:22,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:22,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015042388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:22,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:22,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015042587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:22,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:22,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015042597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:22,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:22,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015042597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:22,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:22,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015042597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:22,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:22,661 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/268b721b842649fba22d492ab1f37e59 2024-11-19T11:16:22,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/aa1b63e2c48a41d58652b8d79411fd67 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/aa1b63e2c48a41d58652b8d79411fd67 2024-11-19T11:16:22,667 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/aa1b63e2c48a41d58652b8d79411fd67, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T11:16:22,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/d5bf016fa74e4562a118442aedcf00fa as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/d5bf016fa74e4562a118442aedcf00fa 2024-11-19T11:16:22,672 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/d5bf016fa74e4562a118442aedcf00fa, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T11:16:22,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/268b721b842649fba22d492ab1f37e59 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/268b721b842649fba22d492ab1f37e59 2024-11-19T11:16:22,676 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/268b721b842649fba22d492ab1f37e59, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T11:16:22,677 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 5fb872772cd7220dc7447361f8e609a9 in 1292ms, sequenceid=48, compaction requested=false 2024-11-19T11:16:22,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:22,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:22,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-19T11:16:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-19T11:16:22,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-19T11:16:22,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6760 sec 2024-11-19T11:16:22,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.6850 sec 2024-11-19T11:16:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:22,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5fb872772cd7220dc7447361f8e609a9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T11:16:22,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=A 2024-11-19T11:16:22,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:22,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=B 2024-11-19T11:16:22,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:22,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=C 2024-11-19T11:16:22,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:22,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/0f5660a155c547cdba4ce9327a4d5b71 is 50, key is test_row_0/A:col10/1732014982263/Put/seqid=0 2024-11-19T11:16:22,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742273_1449 (size=12001) 2024-11-19T11:16:22,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015042948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:22,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:22,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015042953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:22,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:22,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015042954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:22,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:22,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015042956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:22,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:22,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:23,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015043067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:23,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015043067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:23,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015043077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:23,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015043077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:23,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015043277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015043277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:23,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015043298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:23,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:23,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015043298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:23,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/0f5660a155c547cdba4ce9327a4d5b71 2024-11-19T11:16:23,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/945498d6754d4dcba098d016a6d8a077 is 50, key is test_row_0/B:col10/1732014982263/Put/seqid=0 2024-11-19T11:16:23,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742274_1450 (size=12001) 2024-11-19T11:16:23,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015043597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:23,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015043597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:23,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:23,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:23,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015043608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:23,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:23,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015043608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:23,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:23,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:23,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/945498d6754d4dcba098d016a6d8a077 2024-11-19T11:16:23,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/7fe670452e9148528b3b40fa329c10ea is 50, key is test_row_0/C:col10/1732014982263/Put/seqid=0 2024-11-19T11:16:23,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742275_1451 (size=12001) 2024-11-19T11:16:24,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:24,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015044107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:24,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:24,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:24,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:24,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015044118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:24,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:24,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:24,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:24,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015044128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:24,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:24,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:24,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015044128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:24,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:24,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:24,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:24,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:24,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/7fe670452e9148528b3b40fa329c10ea 2024-11-19T11:16:24,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/0f5660a155c547cdba4ce9327a4d5b71 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/0f5660a155c547cdba4ce9327a4d5b71 2024-11-19T11:16:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T11:16:24,168 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:16:24,168 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:16:24,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/0f5660a155c547cdba4ce9327a4d5b71, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T11:16:24,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-19T11:16:24,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/945498d6754d4dcba098d016a6d8a077 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/945498d6754d4dcba098d016a6d8a077 2024-11-19T11:16:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T11:16:24,170 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:16:24,170 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:16:24,170 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:16:24,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/945498d6754d4dcba098d016a6d8a077, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T11:16:24,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/7fe670452e9148528b3b40fa329c10ea as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/7fe670452e9148528b3b40fa329c10ea 2024-11-19T11:16:24,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/7fe670452e9148528b3b40fa329c10ea, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T11:16:24,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5fb872772cd7220dc7447361f8e609a9 in 1266ms, sequenceid=60, compaction requested=true 2024-11-19T11:16:24,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:24,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5fb872772cd7220dc7447361f8e609a9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:16:24,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:24,179 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:24,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5fb872772cd7220dc7447361f8e609a9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:16:24,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:24,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5fb872772cd7220dc7447361f8e609a9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:16:24,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:24,179 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:24,179 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:24,179 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:24,179 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 5fb872772cd7220dc7447361f8e609a9/B is initiating minor compaction (all files) 2024-11-19T11:16:24,179 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 5fb872772cd7220dc7447361f8e609a9/A is initiating minor compaction (all files) 2024-11-19T11:16:24,179 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5fb872772cd7220dc7447361f8e609a9/A in TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:24,179 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5fb872772cd7220dc7447361f8e609a9/B in TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:24,180 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/bd8d18e33ba040d9ac13d31d7a4b10cb, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/d5bf016fa74e4562a118442aedcf00fa, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/945498d6754d4dcba098d016a6d8a077] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp, totalSize=35.2 K 2024-11-19T11:16:24,180 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/a67d2c6cad094fb5b0d0a00ca394c147, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/aa1b63e2c48a41d58652b8d79411fd67, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/0f5660a155c547cdba4ce9327a4d5b71] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp, totalSize=35.2 K 2024-11-19T11:16:24,180 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting bd8d18e33ba040d9ac13d31d7a4b10cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732014979996 2024-11-19T11:16:24,180 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting a67d2c6cad094fb5b0d0a00ca394c147, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732014979996 2024-11-19T11:16:24,180 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting d5bf016fa74e4562a118442aedcf00fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732014980098 2024-11-19T11:16:24,180 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa1b63e2c48a41d58652b8d79411fd67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732014980098 2024-11-19T11:16:24,180 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f5660a155c547cdba4ce9327a4d5b71, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732014982263 2024-11-19T11:16:24,180 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 945498d6754d4dcba098d016a6d8a077, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732014982263 2024-11-19T11:16:24,191 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5fb872772cd7220dc7447361f8e609a9#A#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:24,191 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/bca6061135764191a2a4244b1a06a261 is 50, key is test_row_0/A:col10/1732014982263/Put/seqid=0 2024-11-19T11:16:24,191 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5fb872772cd7220dc7447361f8e609a9#B#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:24,192 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/2e67b88139a544e987fdef351477c579 is 50, key is test_row_0/B:col10/1732014982263/Put/seqid=0 2024-11-19T11:16:24,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742277_1453 (size=12104) 2024-11-19T11:16:24,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742276_1452 (size=12104) 2024-11-19T11:16:24,201 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/bca6061135764191a2a4244b1a06a261 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/bca6061135764191a2a4244b1a06a261 2024-11-19T11:16:24,206 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5fb872772cd7220dc7447361f8e609a9/A of 5fb872772cd7220dc7447361f8e609a9 into bca6061135764191a2a4244b1a06a261(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:24,206 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:24,206 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., storeName=5fb872772cd7220dc7447361f8e609a9/A, priority=13, startTime=1732014984179; duration=0sec 2024-11-19T11:16:24,206 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:24,206 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5fb872772cd7220dc7447361f8e609a9:A 2024-11-19T11:16:24,206 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:24,206 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:24,207 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 5fb872772cd7220dc7447361f8e609a9/C is initiating minor compaction (all files) 2024-11-19T11:16:24,207 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5fb872772cd7220dc7447361f8e609a9/C in TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:24,207 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/5084288f6735469d9678a3830c8e68e4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/268b721b842649fba22d492ab1f37e59, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/7fe670452e9148528b3b40fa329c10ea] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp, totalSize=35.2 K 2024-11-19T11:16:24,207 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5084288f6735469d9678a3830c8e68e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732014979996 2024-11-19T11:16:24,207 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 268b721b842649fba22d492ab1f37e59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732014980098 2024-11-19T11:16:24,207 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fe670452e9148528b3b40fa329c10ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732014982263 2024-11-19T11:16:24,213 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5fb872772cd7220dc7447361f8e609a9#C#compaction#382 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:24,213 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/3fbb106d923d41b49f64d6874616a7e2 is 50, key is test_row_0/C:col10/1732014982263/Put/seqid=0 2024-11-19T11:16:24,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742278_1454 (size=12104) 2024-11-19T11:16:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T11:16:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:24,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014982940, max timestamp=1732014984310Num uniques 859; 2024-11-19T11:16:24,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:24,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014982940, max timestamp=1732014984310Num uniques 859; 2024-11-19T11:16:24,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:24,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014982940, max timestamp=1732014984310Num uniques 859; 2024-11-19T11:16:24,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:24,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5fb872772cd7220dc7447361f8e609a9 3/3 column families, dataSize=33.54 KB heapSize=86.16 KB 2024-11-19T11:16:24,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=A 2024-11-19T11:16:24,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:24,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=B 2024-11-19T11:16:24,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:24,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=C 2024-11-19T11:16:24,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:24,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T11:16:24,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:24,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:24,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:24,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:24,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:24,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/a00d483157a2401ab388ecd1e4883665 is 50, key is test_row_0/A:col10/1732014984318/Put/seqid=0 2024-11-19T11:16:24,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742279_1455 (size=14341) 2024-11-19T11:16:24,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/a00d483157a2401ab388ecd1e4883665 2024-11-19T11:16:24,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/4ec2a229563e4527ad5a8e3181860563 is 50, key is test_row_0/B:col10/1732014984318/Put/seqid=0 2024-11-19T11:16:24,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742280_1456 (size=12001) 2024-11-19T11:16:24,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T11:16:24,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:24,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:24,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:24,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:24,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:24,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T11:16:24,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014984455, max timestamp=1732014984488Num uniques 859; 2024-11-19T11:16:24,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:24,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014984455, max timestamp=1732014984488Num uniques 859; 2024-11-19T11:16:24,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:24,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014984455, max timestamp=1732014984488Num uniques 859; 2024-11-19T11:16:24,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:24,599 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/2e67b88139a544e987fdef351477c579 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/2e67b88139a544e987fdef351477c579 2024-11-19T11:16:24,603 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5fb872772cd7220dc7447361f8e609a9/B of 5fb872772cd7220dc7447361f8e609a9 into 2e67b88139a544e987fdef351477c579(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:24,603 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:24,603 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., storeName=5fb872772cd7220dc7447361f8e609a9/B, priority=13, startTime=1732014984179; duration=0sec 2024-11-19T11:16:24,603 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:24,603 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5fb872772cd7220dc7447361f8e609a9:B 2024-11-19T11:16:24,620 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/3fbb106d923d41b49f64d6874616a7e2 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/3fbb106d923d41b49f64d6874616a7e2 2024-11-19T11:16:24,623 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5fb872772cd7220dc7447361f8e609a9/C of 5fb872772cd7220dc7447361f8e609a9 into 3fbb106d923d41b49f64d6874616a7e2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:24,623 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:24,623 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., storeName=5fb872772cd7220dc7447361f8e609a9/C, priority=13, startTime=1732014984179; duration=0sec 2024-11-19T11:16:24,623 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:24,623 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5fb872772cd7220dc7447361f8e609a9:C 2024-11-19T11:16:24,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T11:16:24,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:24,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:24,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:24,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:24,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:24,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732014984555, max timestamp=1732014984642Num uniques 1018; 2024-11-19T11:16:24,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T11:16:24,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732014984555, max timestamp=1732014984642Num uniques 1018; 2024-11-19T11:16:24,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T11:16:24,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732014984555, max timestamp=1732014984642Num uniques 1018; 2024-11-19T11:16:24,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T11:16:24,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/4ec2a229563e4527ad5a8e3181860563 2024-11-19T11:16:24,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732014984757, max timestamp=1732014984780Num uniques 1027; 2024-11-19T11:16:24,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732014984757, max timestamp=1732014984780Num uniques 1027; 2024-11-19T11:16:24,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732014984757, max timestamp=1732014984780Num uniques 1027; 2024-11-19T11:16:24,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-19T11:16:24,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-19T11:16:24,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-19T11:16:24,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T11:16:24,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:24,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:24,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:24,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:24,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:24,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/9b9e5b4e4479459fa4f7b5648941766f is 50, key is test_row_0/C:col10/1732014984318/Put/seqid=0 2024-11-19T11:16:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T11:16:24,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742281_1457 (size=12001) 2024-11-19T11:16:24,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/9b9e5b4e4479459fa4f7b5648941766f 2024-11-19T11:16:24,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/a00d483157a2401ab388ecd1e4883665 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/a00d483157a2401ab388ecd1e4883665 2024-11-19T11:16:24,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/a00d483157a2401ab388ecd1e4883665, entries=200, sequenceid=83, filesize=14.0 K 2024-11-19T11:16:24,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/4ec2a229563e4527ad5a8e3181860563 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/4ec2a229563e4527ad5a8e3181860563 2024-11-19T11:16:24,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/4ec2a229563e4527ad5a8e3181860563, entries=150, sequenceid=83, filesize=11.7 K 2024-11-19T11:16:24,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/9b9e5b4e4479459fa4f7b5648941766f as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/9b9e5b4e4479459fa4f7b5648941766f 2024-11-19T11:16:24,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/9b9e5b4e4479459fa4f7b5648941766f, entries=150, sequenceid=83, filesize=11.7 K 2024-11-19T11:16:24,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=53.67 KB/54960 for 5fb872772cd7220dc7447361f8e609a9 in 505ms, sequenceid=83, compaction requested=false 2024-11-19T11:16:24,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:24,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5fb872772cd7220dc7447361f8e609a9 3/3 column families, dataSize=60.38 KB heapSize=164.16 KB 2024-11-19T11:16:24,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=A 2024-11-19T11:16:24,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:24,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=B 2024-11-19T11:16:24,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:24,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=C 2024-11-19T11:16:24,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:24,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/eaef461635e44087a3faa7c6dae4fbd4 is 50, key is test_row_0/A:col10/1732014984796/Put/seqid=0 2024-11-19T11:16:24,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742282_1458 (size=14541) 2024-11-19T11:16:24,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T11:16:24,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:24,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:24,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:24,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:24,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:24,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014984946, max timestamp=1732014984985Num uniques 859; 2024-11-19T11:16:24,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014984946, max timestamp=1732014984985Num uniques 859; 2024-11-19T11:16:24,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014984946, max timestamp=1732014984985Num uniques 859; 2024-11-19T11:16:24,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:24,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:24,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:25,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T11:16:25,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:25,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:25,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:25,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:25,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:25,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:25,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015045126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:25,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:25,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:25,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:25,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:25,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015045127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:25,143 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_0' in 5fb872772cd7220dc7447361f8e609a9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:25,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:25,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:25,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:25,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:25,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015045137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:25,149 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_0' in 5fb872772cd7220dc7447361f8e609a9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:25,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:25,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:25,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:25,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:25,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015045147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:25,158 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_0' in 5fb872772cd7220dc7447361f8e609a9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:25,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:25,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:25,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:25,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:25,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015045158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:25,165 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_0' in 5fb872772cd7220dc7447361f8e609a9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:25,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:25,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:25,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:25,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/eaef461635e44087a3faa7c6dae4fbd4 2024-11-19T11:16:25,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:25,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015045237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:25,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:25,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:25,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:25,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T11:16:25,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:25,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:25,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:25,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:25,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:25,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:25,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/3dbc8bd52f9b4a8fac1fa3cc5258be1a is 50, key is test_row_0/B:col10/1732014984796/Put/seqid=0 2024-11-19T11:16:25,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742283_1459 (size=12151) 2024-11-19T11:16:25,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T11:16:25,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T11:16:25,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:25,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:25,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:25,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:25,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:25,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:25,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015045447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:25,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T11:16:25,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:25,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:25,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:25,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:25,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:25,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/3dbc8bd52f9b4a8fac1fa3cc5258be1a 2024-11-19T11:16:25,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/d4b1de313acf4d8bbb42278017486783 is 50, key is test_row_0/C:col10/1732014984796/Put/seqid=0 2024-11-19T11:16:25,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742284_1460 (size=12151) 2024-11-19T11:16:25,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T11:16:25,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:25,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:25,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:25,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:25,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:25,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015045768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:25,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T11:16:25,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:25,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:25,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:25,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:25,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:26,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T11:16:26,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:26,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:26,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:26,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:26,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:26,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/d4b1de313acf4d8bbb42278017486783 2024-11-19T11:16:26,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/eaef461635e44087a3faa7c6dae4fbd4 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/eaef461635e44087a3faa7c6dae4fbd4 2024-11-19T11:16:26,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/eaef461635e44087a3faa7c6dae4fbd4, entries=200, sequenceid=145, filesize=14.2 K 2024-11-19T11:16:26,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/3dbc8bd52f9b4a8fac1fa3cc5258be1a as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/3dbc8bd52f9b4a8fac1fa3cc5258be1a 2024-11-19T11:16:26,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/3dbc8bd52f9b4a8fac1fa3cc5258be1a, entries=150, sequenceid=145, filesize=11.9 K 2024-11-19T11:16:26,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/d4b1de313acf4d8bbb42278017486783 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/d4b1de313acf4d8bbb42278017486783 2024-11-19T11:16:26,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/d4b1de313acf4d8bbb42278017486783, entries=150, sequenceid=145, filesize=11.9 K 2024-11-19T11:16:26,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~164.11 KB/168048, currentSize=127.47 KB/130530 for 5fb872772cd7220dc7447361f8e609a9 in 1239ms, sequenceid=145, compaction requested=true 2024-11-19T11:16:26,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:26,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5fb872772cd7220dc7447361f8e609a9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:16:26,071 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:26,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:26,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5fb872772cd7220dc7447361f8e609a9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:16:26,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:26,071 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:26,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5fb872772cd7220dc7447361f8e609a9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:16:26,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:26,072 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40986 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:26,072 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:26,072 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 5fb872772cd7220dc7447361f8e609a9/A is initiating minor compaction (all files) 2024-11-19T11:16:26,072 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 5fb872772cd7220dc7447361f8e609a9/B is initiating minor compaction (all files) 2024-11-19T11:16:26,072 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5fb872772cd7220dc7447361f8e609a9/A in TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:26,072 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5fb872772cd7220dc7447361f8e609a9/B in TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:26,072 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/bca6061135764191a2a4244b1a06a261, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/a00d483157a2401ab388ecd1e4883665, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/eaef461635e44087a3faa7c6dae4fbd4] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp, totalSize=40.0 K 2024-11-19T11:16:26,072 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/2e67b88139a544e987fdef351477c579, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/4ec2a229563e4527ad5a8e3181860563, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/3dbc8bd52f9b4a8fac1fa3cc5258be1a] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp, totalSize=35.4 K 2024-11-19T11:16:26,072 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e67b88139a544e987fdef351477c579, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732014982263 2024-11-19T11:16:26,072 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting bca6061135764191a2a4244b1a06a261, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732014982263 2024-11-19T11:16:26,073 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ec2a229563e4527ad5a8e3181860563, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732014982940 2024-11-19T11:16:26,073 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting a00d483157a2401ab388ecd1e4883665, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732014982940 2024-11-19T11:16:26,073 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dbc8bd52f9b4a8fac1fa3cc5258be1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732014984771 2024-11-19T11:16:26,073 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting eaef461635e44087a3faa7c6dae4fbd4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732014984771 2024-11-19T11:16:26,077 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5fb872772cd7220dc7447361f8e609a9#A#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:26,078 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/518bd5664b9140b281dc42a3e1b2520a is 50, key is test_row_0/A:col10/1732014984796/Put/seqid=0 2024-11-19T11:16:26,078 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5fb872772cd7220dc7447361f8e609a9#B#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:26,078 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/825a87f285da40059917c03989ce150b is 50, key is test_row_0/B:col10/1732014984796/Put/seqid=0 2024-11-19T11:16:26,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742285_1461 (size=12357) 2024-11-19T11:16:26,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742286_1462 (size=12357) 2024-11-19T11:16:26,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T11:16:26,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:26,161 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 5fb872772cd7220dc7447361f8e609a9 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-19T11:16:26,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=A 2024-11-19T11:16:26,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:26,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=B 2024-11-19T11:16:26,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:26,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=C 2024-11-19T11:16:26,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:26,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/425f830e2dbd4b7b8c0440efce6b3ddd is 50, key is test_row_0/A:col10/1732014985071/Put/seqid=0 2024-11-19T11:16:26,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742287_1463 (size=12151) 2024-11-19T11:16:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:26,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T11:16:26,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015046356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:26,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015046477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:26,486 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/518bd5664b9140b281dc42a3e1b2520a as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/518bd5664b9140b281dc42a3e1b2520a 2024-11-19T11:16:26,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:26,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:26,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:26,489 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5fb872772cd7220dc7447361f8e609a9/A of 5fb872772cd7220dc7447361f8e609a9 into 518bd5664b9140b281dc42a3e1b2520a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:26,489 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:26,489 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., storeName=5fb872772cd7220dc7447361f8e609a9/A, priority=13, startTime=1732014986071; duration=0sec 2024-11-19T11:16:26,489 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:26,489 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5fb872772cd7220dc7447361f8e609a9:A 2024-11-19T11:16:26,489 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:26,493 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:26,493 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 5fb872772cd7220dc7447361f8e609a9/C is initiating minor compaction (all files) 2024-11-19T11:16:26,493 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5fb872772cd7220dc7447361f8e609a9/C in TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:26,493 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/3fbb106d923d41b49f64d6874616a7e2, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/9b9e5b4e4479459fa4f7b5648941766f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/d4b1de313acf4d8bbb42278017486783] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp, totalSize=35.4 K 2024-11-19T11:16:26,494 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fbb106d923d41b49f64d6874616a7e2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732014982263 2024-11-19T11:16:26,495 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/825a87f285da40059917c03989ce150b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/825a87f285da40059917c03989ce150b 2024-11-19T11:16:26,495 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b9e5b4e4479459fa4f7b5648941766f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732014982940 2024-11-19T11:16:26,495 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4b1de313acf4d8bbb42278017486783, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732014984771 2024-11-19T11:16:26,499 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5fb872772cd7220dc7447361f8e609a9/B of 5fb872772cd7220dc7447361f8e609a9 into 825a87f285da40059917c03989ce150b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:26,499 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:26,499 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., storeName=5fb872772cd7220dc7447361f8e609a9/B, priority=13, startTime=1732014986071; duration=0sec 2024-11-19T11:16:26,499 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:26,499 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5fb872772cd7220dc7447361f8e609a9:B 2024-11-19T11:16:26,500 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5fb872772cd7220dc7447361f8e609a9#C#compaction#392 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:26,500 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/2acc03361f7f4162a7e020cc3246ba1f is 50, key is test_row_0/C:col10/1732014984796/Put/seqid=0 2024-11-19T11:16:26,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742288_1464 (size=12357) 2024-11-19T11:16:26,577 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/425f830e2dbd4b7b8c0440efce6b3ddd 2024-11-19T11:16:26,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/8443849a373d4228bb032383ff6ca6c0 is 50, key is test_row_0/B:col10/1732014985071/Put/seqid=0 2024-11-19T11:16:26,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742289_1465 (size=12151) 2024-11-19T11:16:26,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015046697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:26,909 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/2acc03361f7f4162a7e020cc3246ba1f as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/2acc03361f7f4162a7e020cc3246ba1f 2024-11-19T11:16:26,912 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5fb872772cd7220dc7447361f8e609a9/C of 5fb872772cd7220dc7447361f8e609a9 into 2acc03361f7f4162a7e020cc3246ba1f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:26,912 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:26,912 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., storeName=5fb872772cd7220dc7447361f8e609a9/C, priority=13, startTime=1732014986071; duration=0sec 2024-11-19T11:16:26,912 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:26,912 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5fb872772cd7220dc7447361f8e609a9:C 2024-11-19T11:16:26,987 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/8443849a373d4228bb032383ff6ca6c0 2024-11-19T11:16:26,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/679982c3ab52463d9158c30a730bb048 is 50, key is test_row_0/C:col10/1732014985071/Put/seqid=0 2024-11-19T11:16:26,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742290_1466 (size=12151) 2024-11-19T11:16:27,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015047008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:27,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:27,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:27,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:27,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015047157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:27,168 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_0' in 5fb872772cd7220dc7447361f8e609a9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:27,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:27,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:27,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:27,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015047167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:27,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015047168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:27,176 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_0' in 5fb872772cd7220dc7447361f8e609a9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:27,177 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_0' in 5fb872772cd7220dc7447361f8e609a9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:27,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:27,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:27,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:27,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015047178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:27,185 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_0' in 5fb872772cd7220dc7447361f8e609a9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:27,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:27,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:27,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:27,397 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/679982c3ab52463d9158c30a730bb048 2024-11-19T11:16:27,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/425f830e2dbd4b7b8c0440efce6b3ddd as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/425f830e2dbd4b7b8c0440efce6b3ddd 2024-11-19T11:16:27,402 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/425f830e2dbd4b7b8c0440efce6b3ddd, entries=150, sequenceid=181, filesize=11.9 K 2024-11-19T11:16:27,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/8443849a373d4228bb032383ff6ca6c0 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/8443849a373d4228bb032383ff6ca6c0 2024-11-19T11:16:27,405 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/8443849a373d4228bb032383ff6ca6c0, entries=150, sequenceid=181, filesize=11.9 K 2024-11-19T11:16:27,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/679982c3ab52463d9158c30a730bb048 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/679982c3ab52463d9158c30a730bb048 2024-11-19T11:16:27,408 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/679982c3ab52463d9158c30a730bb048, entries=150, sequenceid=181, filesize=11.9 K 2024-11-19T11:16:27,409 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 5fb872772cd7220dc7447361f8e609a9 in 1248ms, sequenceid=181, compaction requested=false 2024-11-19T11:16:27,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:27,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:27,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-19T11:16:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-19T11:16:27,412 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-19T11:16:27,412 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2400 sec 2024-11-19T11:16:27,413 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 3.2440 sec 2024-11-19T11:16:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:27,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5fb872772cd7220dc7447361f8e609a9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T11:16:27,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=A 2024-11-19T11:16:27,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:27,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=B 2024-11-19T11:16:27,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:27,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=C 2024-11-19T11:16:27,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:27,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/ceff646a36ea47229c49d83c22b2d98f is 50, key is test_row_0/A:col10/1732014987531/Put/seqid=0 2024-11-19T11:16:27,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742291_1467 (size=14541) 2024-11-19T11:16:27,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:27,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015047641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:27,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:27,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:27,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:27,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015047748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:27,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:27,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:27,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:27,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/ceff646a36ea47229c49d83c22b2d98f 2024-11-19T11:16:27,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/fe04fc4f34a846e9a219414aaceea98b is 50, key is test_row_0/B:col10/1732014987531/Put/seqid=0 2024-11-19T11:16:27,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742292_1468 (size=12151) 2024-11-19T11:16:27,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015047957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:28,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:28,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015048277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:28,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:28,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:28,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T11:16:28,348 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:16:28,348 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:16:28,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-19T11:16:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T11:16:28,350 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:16:28,350 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:16:28,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:16:28,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/fe04fc4f34a846e9a219414aaceea98b 2024-11-19T11:16:28,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/cd95ca8be94a4021a95fd585b069e774 is 50, key is test_row_0/C:col10/1732014987531/Put/seqid=0 2024-11-19T11:16:28,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742293_1469 (size=12151) 2024-11-19T11:16:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T11:16:28,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T11:16:28,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:28,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:28,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:28,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:28,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:28,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T11:16:28,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:28,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:28,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:28,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:28,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T11:16:28,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/cd95ca8be94a4021a95fd585b069e774 2024-11-19T11:16:28,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/ceff646a36ea47229c49d83c22b2d98f as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/ceff646a36ea47229c49d83c22b2d98f 2024-11-19T11:16:28,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/ceff646a36ea47229c49d83c22b2d98f, entries=200, sequenceid=196, filesize=14.2 K 2024-11-19T11:16:28,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/fe04fc4f34a846e9a219414aaceea98b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/fe04fc4f34a846e9a219414aaceea98b 2024-11-19T11:16:28,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/fe04fc4f34a846e9a219414aaceea98b, entries=150, sequenceid=196, filesize=11.9 K 2024-11-19T11:16:28,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/cd95ca8be94a4021a95fd585b069e774 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/cd95ca8be94a4021a95fd585b069e774 2024-11-19T11:16:28,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/cd95ca8be94a4021a95fd585b069e774, entries=150, sequenceid=196, filesize=11.9 K 2024-11-19T11:16:28,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5fb872772cd7220dc7447361f8e609a9 in 1242ms, sequenceid=196, compaction requested=true 2024-11-19T11:16:28,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:28,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5fb872772cd7220dc7447361f8e609a9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:16:28,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:28,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5fb872772cd7220dc7447361f8e609a9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:16:28,775 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:28,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:28,775 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:28,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5fb872772cd7220dc7447361f8e609a9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:16:28,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:28,775 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:28,775 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:28,775 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 5fb872772cd7220dc7447361f8e609a9/B is initiating minor compaction (all files) 2024-11-19T11:16:28,776 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 5fb872772cd7220dc7447361f8e609a9/A is initiating minor compaction (all files) 2024-11-19T11:16:28,776 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5fb872772cd7220dc7447361f8e609a9/A in TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:28,776 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5fb872772cd7220dc7447361f8e609a9/B in TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:28,776 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/518bd5664b9140b281dc42a3e1b2520a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/425f830e2dbd4b7b8c0440efce6b3ddd, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/ceff646a36ea47229c49d83c22b2d98f] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp, totalSize=38.1 K 2024-11-19T11:16:28,776 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/825a87f285da40059917c03989ce150b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/8443849a373d4228bb032383ff6ca6c0, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/fe04fc4f34a846e9a219414aaceea98b] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp, totalSize=35.8 K 2024-11-19T11:16:28,776 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 518bd5664b9140b281dc42a3e1b2520a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732014984771 2024-11-19T11:16:28,776 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 825a87f285da40059917c03989ce150b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732014984771 2024-11-19T11:16:28,776 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8443849a373d4228bb032383ff6ca6c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732014985071 2024-11-19T11:16:28,776 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 425f830e2dbd4b7b8c0440efce6b3ddd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732014985071 2024-11-19T11:16:28,776 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting fe04fc4f34a846e9a219414aaceea98b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732014986327 2024-11-19T11:16:28,776 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting ceff646a36ea47229c49d83c22b2d98f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732014986327 2024-11-19T11:16:28,781 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5fb872772cd7220dc7447361f8e609a9#B#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:28,782 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/801a824189c84853b1fa7ada12c58029 is 50, key is test_row_0/B:col10/1732014987531/Put/seqid=0 2024-11-19T11:16:28,782 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5fb872772cd7220dc7447361f8e609a9#A#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:28,783 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/363bc246da8e46f087fd4710e25357f8 is 50, key is test_row_0/A:col10/1732014987531/Put/seqid=0 2024-11-19T11:16:28,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742295_1471 (size=12459) 2024-11-19T11:16:28,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742294_1470 (size=12459) 2024-11-19T11:16:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:28,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5fb872772cd7220dc7447361f8e609a9 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-19T11:16:28,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=A 2024-11-19T11:16:28,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:28,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=B 2024-11-19T11:16:28,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:28,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=C 2024-11-19T11:16:28,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:28,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/78c50df82a414006a2395e8a484331c0 is 50, key is test_row_0/A:col10/1732014987627/Put/seqid=0 2024-11-19T11:16:28,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742296_1472 (size=14541) 2024-11-19T11:16:28,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T11:16:28,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:28,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:28,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:28,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:28,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:28,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015048857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:28,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:28,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:28,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:28,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T11:16:28,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:28,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:28,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:28,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:28,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:28,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:28,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:28,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015048968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:28,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T11:16:28,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:28,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:28,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:29,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T11:16:29,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:29,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:29,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:29,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:29,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:29,190 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/801a824189c84853b1fa7ada12c58029 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/801a824189c84853b1fa7ada12c58029 2024-11-19T11:16:29,190 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/363bc246da8e46f087fd4710e25357f8 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/363bc246da8e46f087fd4710e25357f8 2024-11-19T11:16:29,194 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5fb872772cd7220dc7447361f8e609a9/A of 5fb872772cd7220dc7447361f8e609a9 into 363bc246da8e46f087fd4710e25357f8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:29,194 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5fb872772cd7220dc7447361f8e609a9/B of 5fb872772cd7220dc7447361f8e609a9 into 801a824189c84853b1fa7ada12c58029(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:29,194 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:29,194 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:29,194 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., storeName=5fb872772cd7220dc7447361f8e609a9/B, priority=13, startTime=1732014988775; duration=0sec 2024-11-19T11:16:29,194 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., storeName=5fb872772cd7220dc7447361f8e609a9/A, priority=13, startTime=1732014988774; duration=0sec 2024-11-19T11:16:29,194 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:29,194 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:29,194 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5fb872772cd7220dc7447361f8e609a9:B 2024-11-19T11:16:29,194 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5fb872772cd7220dc7447361f8e609a9:A 2024-11-19T11:16:29,194 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:29,194 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:29,195 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 5fb872772cd7220dc7447361f8e609a9/C is initiating minor compaction (all files) 2024-11-19T11:16:29,195 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5fb872772cd7220dc7447361f8e609a9/C in TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:29,195 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/2acc03361f7f4162a7e020cc3246ba1f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/679982c3ab52463d9158c30a730bb048, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/cd95ca8be94a4021a95fd585b069e774] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp, totalSize=35.8 K 2024-11-19T11:16:29,195 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 2acc03361f7f4162a7e020cc3246ba1f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732014984771 2024-11-19T11:16:29,195 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 679982c3ab52463d9158c30a730bb048, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732014985071 2024-11-19T11:16:29,195 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting cd95ca8be94a4021a95fd585b069e774, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732014986327 2024-11-19T11:16:29,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015049187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:29,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:29,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:29,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:29,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/78c50df82a414006a2395e8a484331c0 2024-11-19T11:16:29,201 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5fb872772cd7220dc7447361f8e609a9#C#compaction#401 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:29,201 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/8bfce21705724dbcacae2426edccb506 is 50, key is test_row_0/C:col10/1732014987531/Put/seqid=0 2024-11-19T11:16:29,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/28333748a34e4e9b9300684ee64dae5f is 50, key is test_row_0/B:col10/1732014987627/Put/seqid=0 2024-11-19T11:16:29,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742297_1473 (size=12459) 2024-11-19T11:16:29,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742298_1474 (size=12151) 2024-11-19T11:16:29,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T11:16:29,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:29,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:29,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:29,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:29,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:29,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T11:16:29,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:29,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:29,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:29,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:29,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T11:16:29,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015049507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:29,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:29,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:29,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:29,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T11:16:29,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:29,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:29,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:29,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:29,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:29,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/28333748a34e4e9b9300684ee64dae5f 2024-11-19T11:16:29,618 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/8bfce21705724dbcacae2426edccb506 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/8bfce21705724dbcacae2426edccb506 2024-11-19T11:16:29,624 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5fb872772cd7220dc7447361f8e609a9/C of 5fb872772cd7220dc7447361f8e609a9 into 8bfce21705724dbcacae2426edccb506(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:29,624 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:29,624 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., storeName=5fb872772cd7220dc7447361f8e609a9/C, priority=13, startTime=1732014988775; duration=0sec 2024-11-19T11:16:29,624 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:29,624 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5fb872772cd7220dc7447361f8e609a9:C 2024-11-19T11:16:29,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/94efb2e1f5494d2a8ae664205fbbe5c5 is 50, key is test_row_0/C:col10/1732014987627/Put/seqid=0 2024-11-19T11:16:29,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742299_1475 (size=12151) 2024-11-19T11:16:29,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T11:16:29,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:29,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:29,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:29,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:29,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:29,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T11:16:29,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:29,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:29,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:29,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:29,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:30,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T11:16:30,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:30,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:30,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:30,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:30,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:30,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:30,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/94efb2e1f5494d2a8ae664205fbbe5c5 2024-11-19T11:16:30,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015050027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:30,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:30,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:30,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:30,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/78c50df82a414006a2395e8a484331c0 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/78c50df82a414006a2395e8a484331c0 2024-11-19T11:16:30,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/78c50df82a414006a2395e8a484331c0, entries=200, sequenceid=217, filesize=14.2 K 2024-11-19T11:16:30,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/28333748a34e4e9b9300684ee64dae5f as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/28333748a34e4e9b9300684ee64dae5f 2024-11-19T11:16:30,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/28333748a34e4e9b9300684ee64dae5f, entries=150, sequenceid=217, filesize=11.9 K 2024-11-19T11:16:30,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/94efb2e1f5494d2a8ae664205fbbe5c5 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/94efb2e1f5494d2a8ae664205fbbe5c5 2024-11-19T11:16:30,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/94efb2e1f5494d2a8ae664205fbbe5c5, entries=150, sequenceid=217, filesize=11.9 K 2024-11-19T11:16:30,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 5fb872772cd7220dc7447361f8e609a9 in 1252ms, sequenceid=217, compaction requested=false 2024-11-19T11:16:30,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:30,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T11:16:30,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:30,182 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 5fb872772cd7220dc7447361f8e609a9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T11:16:30,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=A 2024-11-19T11:16:30,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:30,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=B 2024-11-19T11:16:30,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:30,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=C 2024-11-19T11:16:30,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:30,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/4a4fd6aa21ef455f88304fcee7299be4 is 50, key is test_row_0/A:col10/1732014988856/Put/seqid=0 2024-11-19T11:16:30,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742300_1476 (size=12151) 2024-11-19T11:16:30,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T11:16:30,590 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/4a4fd6aa21ef455f88304fcee7299be4 2024-11-19T11:16:30,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/05532c51a2fc423d9e14daa3f43f70ca is 50, key is test_row_0/B:col10/1732014988856/Put/seqid=0 2024-11-19T11:16:30,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742301_1477 (size=12151) 2024-11-19T11:16:31,002 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/05532c51a2fc423d9e14daa3f43f70ca 2024-11-19T11:16:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/66f501ea995548af8421d8a3fe0ae7f2 is 50, key is test_row_0/C:col10/1732014988856/Put/seqid=0 2024-11-19T11:16:31,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742302_1478 (size=12151) 2024-11-19T11:16:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:31,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:31,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014991150, max timestamp=1732014991174Num uniques 859; 2024-11-19T11:16:31,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014991150, max timestamp=1732014991174Num uniques 859; 2024-11-19T11:16:31,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014991150, max timestamp=1732014991174Num uniques 859; 2024-11-19T11:16:31,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:31,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:31,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:31,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015051225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:31,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:31,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015051225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:31,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:31,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:31,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015051226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:31,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:31,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015051230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:31,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:31,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015051234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:31,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015051337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:31,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:31,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015051337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:31,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:31,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015051337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:31,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:31,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015051348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:31,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:31,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015051348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:31,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:31,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:31,411 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/66f501ea995548af8421d8a3fe0ae7f2 2024-11-19T11:16:31,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/4a4fd6aa21ef455f88304fcee7299be4 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/4a4fd6aa21ef455f88304fcee7299be4 2024-11-19T11:16:31,417 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/4a4fd6aa21ef455f88304fcee7299be4, entries=150, sequenceid=231, filesize=11.9 K 2024-11-19T11:16:31,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/05532c51a2fc423d9e14daa3f43f70ca as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/05532c51a2fc423d9e14daa3f43f70ca 2024-11-19T11:16:31,420 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/05532c51a2fc423d9e14daa3f43f70ca, entries=150, sequenceid=231, filesize=11.9 K 2024-11-19T11:16:31,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/66f501ea995548af8421d8a3fe0ae7f2 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/66f501ea995548af8421d8a3fe0ae7f2 2024-11-19T11:16:31,423 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/66f501ea995548af8421d8a3fe0ae7f2, entries=150, sequenceid=231, filesize=11.9 K 2024-11-19T11:16:31,424 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 5fb872772cd7220dc7447361f8e609a9 in 1242ms, sequenceid=231, compaction requested=true 2024-11-19T11:16:31,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:31,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:31,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-19T11:16:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-19T11:16:31,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-19T11:16:31,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0750 sec 2024-11-19T11:16:31,427 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 3.0780 sec 2024-11-19T11:16:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:31,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5fb872772cd7220dc7447361f8e609a9 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-19T11:16:31,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732014991219, max timestamp=1732014991558Num uniques 1018; 2024-11-19T11:16:31,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732014991219, max timestamp=1732014991558Num uniques 1018; 2024-11-19T11:16:31,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T11:16:31,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T11:16:31,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732014991219, max timestamp=1732014991558Num uniques 1018; 2024-11-19T11:16:31,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T11:16:31,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=A 2024-11-19T11:16:31,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:31,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=B 2024-11-19T11:16:31,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:31,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=C 2024-11-19T11:16:31,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:31,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/e07b7409f0ec4b248e3c033738e83d6e is 50, key is test_row_0/A:col10/1732014991558/Put/seqid=0 2024-11-19T11:16:31,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742303_1479 (size=17181) 2024-11-19T11:16:31,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732014991596, max timestamp=1732014991612Num uniques 859; 2024-11-19T11:16:31,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T11:16:31,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732014991596, max timestamp=1732014991612Num uniques 859; 2024-11-19T11:16:31,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T11:16:31,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732014991596, max timestamp=1732014991612Num uniques 859; 2024-11-19T11:16:31,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T11:16:31,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732014991686, max timestamp=1732014991703Num uniques 1068; 2024-11-19T11:16:31,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732014991686, max timestamp=1732014991703Num uniques 1068; 2024-11-19T11:16:31,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-19T11:16:31,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-19T11:16:31,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732014991686, max timestamp=1732014991703Num uniques 1068; 2024-11-19T11:16:31,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-19T11:16:31,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015051751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:31,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015051751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:31,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015051763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:31,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,766 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:31,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:31,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732014991748, max timestamp=1732014991763Num uniques 1127; 2024-11-19T11:16:31,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-19T11:16:31,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732014991748, max timestamp=1732014991763Num uniques 1127; 2024-11-19T11:16:31,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-19T11:16:31,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732014991748, max timestamp=1732014991763Num uniques 1127; 2024-11-19T11:16:31,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-19T11:16:31,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:31,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732014991831, max timestamp=1732014991846Num uniques 1086; 2024-11-19T11:16:31,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732014991831, max timestamp=1732014991846Num uniques 1086; 2024-11-19T11:16:31,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-19T11:16:31,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-19T11:16:31,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732014991831, max timestamp=1732014991846Num uniques 1086; 2024-11-19T11:16:31,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-19T11:16:31,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732014991898, max timestamp=1732014991926Num uniques 1095; 2024-11-19T11:16:31,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-19T11:16:31,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732014991898, max timestamp=1732014991926Num uniques 1095; 2024-11-19T11:16:31,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-19T11:16:31,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732014991898, max timestamp=1732014991926Num uniques 1095; 2024-11-19T11:16:31,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-19T11:16:31,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/e07b7409f0ec4b248e3c033738e83d6e 2024-11-19T11:16:31,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/520f0804f22d4264b3adc7970223cb10 is 50, key is test_row_0/B:col10/1732014991558/Put/seqid=0 2024-11-19T11:16:31,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:31,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015051973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:31,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:31,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:31,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11624, totalHeapSize=26656, min timestamp=1732014991941, max timestamp=1732014991979Num uniques 1154; 2024-11-19T11:16:31,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52844, new segment data size=11624, suffix heap size=147984,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=254 2024-11-19T11:16:31,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1732014991941, max timestamp=1732014991979Num uniques 1154; 2024-11-19T11:16:31,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11625, totalHeapSize=26656, min timestamp=1732014991941, max timestamp=1732014991979Num uniques 1154; 2024-11-19T11:16:31,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=11630, suffix heap size=147984,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=254 2024-11-19T11:16:31,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52845, new segment data size=11625, suffix heap size=147984,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=254 2024-11-19T11:16:31,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742304_1480 (size=12301) 2024-11-19T11:16:32,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1732014992017, max timestamp=1732014992036Num uniques 1113; 2024-11-19T11:16:32,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-11-19T11:16:32,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1732014992017, max timestamp=1732014992036Num uniques 1113; 2024-11-19T11:16:32,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-11-19T11:16:32,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1732014992017, max timestamp=1732014992036Num uniques 1113; 2024-11-19T11:16:32,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-11-19T11:16:32,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:32,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015052087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:32,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:32,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015052088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:32,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:32,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1732014992078, max timestamp=1732014992099Num uniques 1122; 2024-11-19T11:16:32,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=12452, suffix heap size=144656,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=272 2024-11-19T11:16:32,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1732014992078, max timestamp=1732014992099Num uniques 1122; 2024-11-19T11:16:32,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=12452, suffix heap size=144656,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=272 2024-11-19T11:16:32,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1732014992078, max timestamp=1732014992099Num uniques 1122; 2024-11-19T11:16:32,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=12452, suffix heap size=144656,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=272 2024-11-19T11:16:32,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:32,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=331, cellSize=15156, totalHeapSize=34664, min timestamp=1732014992141, max timestamp=1732014992161Num uniques 1131; 2024-11-19T11:16:32,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=331, cellSize=15156, totalHeapSize=34664, min timestamp=1732014992141, max timestamp=1732014992161Num uniques 1131; 2024-11-19T11:16:32,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=15156, suffix heap size=145592,new segment heap size=34664 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=331 2024-11-19T11:16:32,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=15156, suffix heap size=145592,new segment heap size=34664 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=331 2024-11-19T11:16:32,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=331, cellSize=15156, totalHeapSize=34664, min timestamp=1732014992141, max timestamp=1732014992161Num uniques 1131; 2024-11-19T11:16:32,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=15156, suffix heap size=145592,new segment heap size=34664 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=331 2024-11-19T11:16:32,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015052220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:32,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1732014992187, max timestamp=1732014992225Num uniques 1190; 2024-11-19T11:16:32,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=54500, new segment data size=15570, suffix heap size=151728,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1190, new segment cells count=340 2024-11-19T11:16:32,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1732014992187, max timestamp=1732014992225Num uniques 1190; 2024-11-19T11:16:32,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=54500, new segment data size=15570, suffix heap size=151728,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1190, new segment cells count=340 2024-11-19T11:16:32,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1732014992187, max timestamp=1732014992225Num uniques 1190; 2024-11-19T11:16:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=54500, new segment data size=15570, suffix heap size=151728,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1190, new segment cells count=340 2024-11-19T11:16:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:32,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:32,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015052284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:32,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1732014992267, max timestamp=1732014992291Num uniques 1199; 2024-11-19T11:16:32,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=13694, suffix heap size=152664,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=299 2024-11-19T11:16:32,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1732014992267, max timestamp=1732014992291Num uniques 1199; 2024-11-19T11:16:32,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=13694, suffix heap size=152664,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=299 2024-11-19T11:16:32,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1732014992267, max timestamp=1732014992291Num uniques 1199; 2024-11-19T11:16:32,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=13694, suffix heap size=152664,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=299 2024-11-19T11:16:32,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:32,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:32,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015052342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:32,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:32,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015052343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:32,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:32,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:32,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:32,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015052349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:32,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015052349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:32,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/520f0804f22d4264b3adc7970223cb10 2024-11-19T11:16:32,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/9be08b2786284c03bcc3b2d4aa4f2c76 is 50, key is test_row_0/C:col10/1732014991558/Put/seqid=0 2024-11-19T11:16:32,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742305_1481 (size=12301) 2024-11-19T11:16:32,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015052397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:32,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:32,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015052458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:32,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015052458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:32,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015052467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:32,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015052467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:32,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T11:16:32,508 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:16:32,508 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:16:32,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-19T11:16:32,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T11:16:32,510 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:16:32,510 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:16:32,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:16:32,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T11:16:32,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015052617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:32,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T11:16:32,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:32,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:32,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:32,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:32,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:32,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:32,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015052677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:32,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:32,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:32,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015052677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:32,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:32,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:32,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015052677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:32,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:32,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015052677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:32,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:32,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:32,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/9be08b2786284c03bcc3b2d4aa4f2c76 2024-11-19T11:16:32,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/e07b7409f0ec4b248e3c033738e83d6e as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/e07b7409f0ec4b248e3c033738e83d6e 2024-11-19T11:16:32,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/e07b7409f0ec4b248e3c033738e83d6e, entries=250, sequenceid=269, filesize=16.8 K 2024-11-19T11:16:32,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/520f0804f22d4264b3adc7970223cb10 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/520f0804f22d4264b3adc7970223cb10 2024-11-19T11:16:32,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/520f0804f22d4264b3adc7970223cb10, entries=150, sequenceid=269, filesize=12.0 K 2024-11-19T11:16:32,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/9be08b2786284c03bcc3b2d4aa4f2c76 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/9be08b2786284c03bcc3b2d4aa4f2c76 2024-11-19T11:16:32,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/9be08b2786284c03bcc3b2d4aa4f2c76, entries=150, sequenceid=269, filesize=12.0 K 2024-11-19T11:16:32,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~80.58 KB/82512, currentSize=154.31 KB/158010 for 5fb872772cd7220dc7447361f8e609a9 in 1248ms, sequenceid=269, compaction requested=true 2024-11-19T11:16:32,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:32,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5fb872772cd7220dc7447361f8e609a9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:16:32,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:32,807 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:16:32,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5fb872772cd7220dc7447361f8e609a9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:16:32,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:32,808 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:16:32,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5fb872772cd7220dc7447361f8e609a9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:16:32,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:32,808 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56332 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:16:32,808 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49062 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:16:32,808 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 5fb872772cd7220dc7447361f8e609a9/B is initiating minor compaction (all files) 2024-11-19T11:16:32,808 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 5fb872772cd7220dc7447361f8e609a9/A is initiating minor compaction (all files) 2024-11-19T11:16:32,808 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5fb872772cd7220dc7447361f8e609a9/A in TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:32,808 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5fb872772cd7220dc7447361f8e609a9/B in TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:32,808 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/363bc246da8e46f087fd4710e25357f8, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/78c50df82a414006a2395e8a484331c0, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/4a4fd6aa21ef455f88304fcee7299be4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/e07b7409f0ec4b248e3c033738e83d6e] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp, totalSize=55.0 K 2024-11-19T11:16:32,808 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/801a824189c84853b1fa7ada12c58029, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/28333748a34e4e9b9300684ee64dae5f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/05532c51a2fc423d9e14daa3f43f70ca, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/520f0804f22d4264b3adc7970223cb10] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp, totalSize=47.9 K 2024-11-19T11:16:32,809 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 801a824189c84853b1fa7ada12c58029, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732014986327 2024-11-19T11:16:32,809 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 363bc246da8e46f087fd4710e25357f8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732014986327 2024-11-19T11:16:32,809 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78c50df82a414006a2395e8a484331c0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732014987618 2024-11-19T11:16:32,809 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 28333748a34e4e9b9300684ee64dae5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732014987627 2024-11-19T11:16:32,809 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a4fd6aa21ef455f88304fcee7299be4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732014988804 2024-11-19T11:16:32,809 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 05532c51a2fc423d9e14daa3f43f70ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732014988804 2024-11-19T11:16:32,809 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting e07b7409f0ec4b248e3c033738e83d6e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732014991219 2024-11-19T11:16:32,809 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 520f0804f22d4264b3adc7970223cb10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732014991233 2024-11-19T11:16:32,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T11:16:32,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:32,815 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 5fb872772cd7220dc7447361f8e609a9 3/3 column families, dataSize=154.31 KB heapSize=431.63 KB 2024-11-19T11:16:32,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=A 2024-11-19T11:16:32,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:32,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=B 2024-11-19T11:16:32,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:32,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=C 2024-11-19T11:16:32,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:32,816 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5fb872772cd7220dc7447361f8e609a9#B#compaction#410 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:32,816 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5fb872772cd7220dc7447361f8e609a9#A#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:32,817 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/9ac0e7970bac49548cf04a08875a22c5 is 50, key is test_row_0/B:col10/1732014991558/Put/seqid=0 2024-11-19T11:16:32,817 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/87fea6ec65a04bd59c707713234b2cfa is 50, key is test_row_0/A:col10/1732014991558/Put/seqid=0 2024-11-19T11:16:32,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/3f75c14ab85b4eb9a7991c9dc29ac112 is 50, key is test_row_0/A:col10/1732014992347/Put/seqid=0 2024-11-19T11:16:32,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742306_1482 (size=12745) 2024-11-19T11:16:32,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742307_1483 (size=12745) 2024-11-19T11:16:32,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742308_1484 (size=12301) 2024-11-19T11:16:32,824 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/3f75c14ab85b4eb9a7991c9dc29ac112 2024-11-19T11:16:32,826 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/9ac0e7970bac49548cf04a08875a22c5 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/9ac0e7970bac49548cf04a08875a22c5 2024-11-19T11:16:32,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T11:16:32,830 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5fb872772cd7220dc7447361f8e609a9/B of 5fb872772cd7220dc7447361f8e609a9 into 9ac0e7970bac49548cf04a08875a22c5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:32,830 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:32,830 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., storeName=5fb872772cd7220dc7447361f8e609a9/B, priority=12, startTime=1732014992807; duration=0sec 2024-11-19T11:16:32,830 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:32,830 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5fb872772cd7220dc7447361f8e609a9:B 2024-11-19T11:16:32,830 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:16:32,831 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49062 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:16:32,831 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 5fb872772cd7220dc7447361f8e609a9/C is initiating minor compaction (all files) 2024-11-19T11:16:32,831 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5fb872772cd7220dc7447361f8e609a9/C in TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:32,831 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/8bfce21705724dbcacae2426edccb506, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/94efb2e1f5494d2a8ae664205fbbe5c5, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/66f501ea995548af8421d8a3fe0ae7f2, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/9be08b2786284c03bcc3b2d4aa4f2c76] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp, totalSize=47.9 K 2024-11-19T11:16:32,831 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bfce21705724dbcacae2426edccb506, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732014986327 2024-11-19T11:16:32,831 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 94efb2e1f5494d2a8ae664205fbbe5c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732014987627 2024-11-19T11:16:32,831 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 66f501ea995548af8421d8a3fe0ae7f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732014988804 2024-11-19T11:16:32,832 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 9be08b2786284c03bcc3b2d4aa4f2c76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732014991233 2024-11-19T11:16:32,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/b8f7356f3e3b400081dc11d24614d3df is 50, key is test_row_0/B:col10/1732014992347/Put/seqid=0 2024-11-19T11:16:32,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742309_1485 (size=12301) 2024-11-19T11:16:32,836 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/b8f7356f3e3b400081dc11d24614d3df 2024-11-19T11:16:32,838 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5fb872772cd7220dc7447361f8e609a9#C#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:32,838 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/b87b8d232ac74212bfe4e9788f9cc8c1 is 50, key is test_row_0/C:col10/1732014991558/Put/seqid=0 2024-11-19T11:16:32,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/21d477e0461a4745863ff16453428389 is 50, key is test_row_0/C:col10/1732014992347/Put/seqid=0 2024-11-19T11:16:32,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742310_1486 (size=12745) 2024-11-19T11:16:32,857 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/b87b8d232ac74212bfe4e9788f9cc8c1 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/b87b8d232ac74212bfe4e9788f9cc8c1 2024-11-19T11:16:32,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742311_1487 (size=12301) 2024-11-19T11:16:32,861 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5fb872772cd7220dc7447361f8e609a9/C of 5fb872772cd7220dc7447361f8e609a9 into b87b8d232ac74212bfe4e9788f9cc8c1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:32,861 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:32,861 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., storeName=5fb872772cd7220dc7447361f8e609a9/C, priority=12, startTime=1732014992808; duration=0sec 2024-11-19T11:16:32,861 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:32,861 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5fb872772cd7220dc7447361f8e609a9:C 2024-11-19T11:16:32,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:32,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015052979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:33,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015052988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:33,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015052997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:33,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015052997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:33,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015052997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:33,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:33,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015053087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:33,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T11:16:33,226 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/87fea6ec65a04bd59c707713234b2cfa as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/87fea6ec65a04bd59c707713234b2cfa 2024-11-19T11:16:33,229 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5fb872772cd7220dc7447361f8e609a9/A of 5fb872772cd7220dc7447361f8e609a9 into 87fea6ec65a04bd59c707713234b2cfa(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:33,229 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:33,229 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., storeName=5fb872772cd7220dc7447361f8e609a9/A, priority=12, startTime=1732014992807; duration=0sec 2024-11-19T11:16:33,229 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:33,229 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5fb872772cd7220dc7447361f8e609a9:A 2024-11-19T11:16:33,259 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/21d477e0461a4745863ff16453428389 2024-11-19T11:16:33,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/3f75c14ab85b4eb9a7991c9dc29ac112 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/3f75c14ab85b4eb9a7991c9dc29ac112 2024-11-19T11:16:33,266 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/3f75c14ab85b4eb9a7991c9dc29ac112, entries=150, sequenceid=478, filesize=12.0 K 2024-11-19T11:16:33,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/b8f7356f3e3b400081dc11d24614d3df as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/b8f7356f3e3b400081dc11d24614d3df 2024-11-19T11:16:33,269 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/b8f7356f3e3b400081dc11d24614d3df, entries=150, sequenceid=478, filesize=12.0 K 2024-11-19T11:16:33,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/21d477e0461a4745863ff16453428389 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/21d477e0461a4745863ff16453428389 2024-11-19T11:16:33,272 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/21d477e0461a4745863ff16453428389, entries=150, sequenceid=478, filesize=12.0 K 2024-11-19T11:16:33,273 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~431.58 KB/441936, currentSize=26.84 KB/27480 for 5fb872772cd7220dc7447361f8e609a9 in 457ms, sequenceid=478, compaction requested=false 2024-11-19T11:16:33,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:33,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:33,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-19T11:16:33,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-19T11:16:33,275 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-19T11:16:33,275 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 764 msec 2024-11-19T11:16:33,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 767 msec 2024-11-19T11:16:33,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:33,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5fb872772cd7220dc7447361f8e609a9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T11:16:33,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=A 2024-11-19T11:16:33,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:33,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=B 2024-11-19T11:16:33,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:33,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=C 2024-11-19T11:16:33,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:33,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/17c545cf58cf465c9db263a8d33e0d92 is 50, key is test_row_1/A:col10/1732014993305/Put/seqid=0 2024-11-19T11:16:33,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742312_1488 (size=12297) 2024-11-19T11:16:33,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014993378, max timestamp=1732014993411Num uniques 859; 2024-11-19T11:16:33,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014993378, max timestamp=1732014993411Num uniques 859; 2024-11-19T11:16:33,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:33,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:33,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732014993378, max timestamp=1732014993411Num uniques 859; 2024-11-19T11:16:33,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:33,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:33,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015053513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:33,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:33,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:33,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015053518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:33,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:33,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015053518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:33,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:33,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:33,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015053518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:33,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:33,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:33,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015053519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:33,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:33,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:33,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:33,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015053627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:33,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:33,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015053627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:33,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:33,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:33,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T11:16:33,648 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:16:33,648 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:16:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-19T11:16:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T11:16:33,649 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:16:33,650 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:16:33,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:16:33,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/17c545cf58cf465c9db263a8d33e0d92 2024-11-19T11:16:33,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/7522d8e8ba5c47bdae2501c375863a57 is 50, key is test_row_1/B:col10/1732014993305/Put/seqid=0 2024-11-19T11:16:33,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742313_1489 (size=9857) 2024-11-19T11:16:33,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T11:16:33,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T11:16:33,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:33,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:33,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:33,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:33,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:33,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:33,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015053837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015053837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:33,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T11:16:33,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:33,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:33,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:33,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:33,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T11:16:34,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T11:16:34,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:34,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:34,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:34,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:34,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:34,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:34,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/7522d8e8ba5c47bdae2501c375863a57 2024-11-19T11:16:34,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/8e2f5d662c3b44e5a2f6a104198efdb8 is 50, key is test_row_1/C:col10/1732014993305/Put/seqid=0 2024-11-19T11:16:34,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742314_1490 (size=9857) 2024-11-19T11:16:34,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:34,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015054148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:34,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015054148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:34,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T11:16:34,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:34,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:34,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:34,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:34,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:34,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:34,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T11:16:34,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T11:16:34,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:34,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:34,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:34,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:34,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:34,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:34,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/8e2f5d662c3b44e5a2f6a104198efdb8 2024-11-19T11:16:34,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/17c545cf58cf465c9db263a8d33e0d92 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/17c545cf58cf465c9db263a8d33e0d92 2024-11-19T11:16:34,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/17c545cf58cf465c9db263a8d33e0d92, entries=150, sequenceid=491, filesize=12.0 K 2024-11-19T11:16:34,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/7522d8e8ba5c47bdae2501c375863a57 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/7522d8e8ba5c47bdae2501c375863a57 2024-11-19T11:16:34,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/7522d8e8ba5c47bdae2501c375863a57, entries=100, sequenceid=491, filesize=9.6 K 2024-11-19T11:16:34,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/8e2f5d662c3b44e5a2f6a104198efdb8 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/8e2f5d662c3b44e5a2f6a104198efdb8 2024-11-19T11:16:34,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/8e2f5d662c3b44e5a2f6a104198efdb8, entries=100, sequenceid=491, filesize=9.6 K 2024-11-19T11:16:34,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 5fb872772cd7220dc7447361f8e609a9 in 1238ms, sequenceid=491, compaction requested=true 2024-11-19T11:16:34,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:34,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5fb872772cd7220dc7447361f8e609a9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:16:34,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:34,544 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:34,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5fb872772cd7220dc7447361f8e609a9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:16:34,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:34,544 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:34,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5fb872772cd7220dc7447361f8e609a9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:16:34,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:34,544 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:34,545 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 5fb872772cd7220dc7447361f8e609a9/B is initiating minor compaction (all files) 2024-11-19T11:16:34,545 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5fb872772cd7220dc7447361f8e609a9/B in TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:34,545 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:34,545 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/9ac0e7970bac49548cf04a08875a22c5, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/b8f7356f3e3b400081dc11d24614d3df, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/7522d8e8ba5c47bdae2501c375863a57] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp, totalSize=34.1 K 2024-11-19T11:16:34,545 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 5fb872772cd7220dc7447361f8e609a9/A is initiating minor compaction (all files) 2024-11-19T11:16:34,545 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5fb872772cd7220dc7447361f8e609a9/A in TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:34,545 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/87fea6ec65a04bd59c707713234b2cfa, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/3f75c14ab85b4eb9a7991c9dc29ac112, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/17c545cf58cf465c9db263a8d33e0d92] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp, totalSize=36.5 K 2024-11-19T11:16:34,545 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ac0e7970bac49548cf04a08875a22c5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732014991233 2024-11-19T11:16:34,545 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87fea6ec65a04bd59c707713234b2cfa, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732014991233 2024-11-19T11:16:34,545 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting b8f7356f3e3b400081dc11d24614d3df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1732014992342 2024-11-19T11:16:34,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5fb872772cd7220dc7447361f8e609a9 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-19T11:16:34,545 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 7522d8e8ba5c47bdae2501c375863a57, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1732014993300 2024-11-19T11:16:34,545 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f75c14ab85b4eb9a7991c9dc29ac112, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1732014992342 2024-11-19T11:16:34,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=A 2024-11-19T11:16:34,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T11:16:34,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=B 2024-11-19T11:16:34,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T11:16:34,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=C 2024-11-19T11:16:34,546 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17c545cf58cf465c9db263a8d33e0d92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1732014993300 2024-11-19T11:16:34,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T11:16:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:34,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/962fe14ed14f4a62a25cac852198e41e is 50, key is test_row_0/A:col10/1732014993504/Put/seqid=0 2024-11-19T11:16:34,552 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5fb872772cd7220dc7447361f8e609a9#B#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:34,552 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/1fee37e04f96485a99cfe874e8151bec is 50, key is test_row_0/B:col10/1732014992347/Put/seqid=0 2024-11-19T11:16:34,553 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5fb872772cd7220dc7447361f8e609a9#A#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:34,553 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/a9571c9bc1c548f383dd1103a38c58b7 is 50, key is test_row_0/A:col10/1732014992347/Put/seqid=0 2024-11-19T11:16:34,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742315_1491 (size=12847) 2024-11-19T11:16:34,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742317_1493 (size=12847) 2024-11-19T11:16:34,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742316_1492 (size=14741) 2024-11-19T11:16:34,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T11:16:34,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:34,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:34,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:34,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:34,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:34,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:34,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015054563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015054563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:34,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:34,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015054568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:34,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015054657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:34,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015054657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:34,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015054677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015054677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:34,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:34,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015054688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:34,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:34,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T11:16:34,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:34,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:34,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:34,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:34,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T11:16:34,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T11:16:34,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:34,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:34,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:34,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:34,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:34,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015054897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:34,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:34,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015054898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:34,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:34,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:34,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015054898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:34,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:34,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/962fe14ed14f4a62a25cac852198e41e 2024-11-19T11:16:34,960 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/a9571c9bc1c548f383dd1103a38c58b7 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/a9571c9bc1c548f383dd1103a38c58b7 2024-11-19T11:16:34,960 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/1fee37e04f96485a99cfe874e8151bec as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/1fee37e04f96485a99cfe874e8151bec 2024-11-19T11:16:34,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/d1cedb97a82148a287e5282d8baf411c is 50, key is test_row_0/B:col10/1732014993504/Put/seqid=0 2024-11-19T11:16:34,967 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5fb872772cd7220dc7447361f8e609a9/B of 5fb872772cd7220dc7447361f8e609a9 into 1fee37e04f96485a99cfe874e8151bec(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:34,967 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:34,967 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5fb872772cd7220dc7447361f8e609a9/A of 5fb872772cd7220dc7447361f8e609a9 into a9571c9bc1c548f383dd1103a38c58b7(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:34,967 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., storeName=5fb872772cd7220dc7447361f8e609a9/B, priority=13, startTime=1732014994544; duration=0sec 2024-11-19T11:16:34,967 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:34,967 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., storeName=5fb872772cd7220dc7447361f8e609a9/A, priority=13, startTime=1732014994544; duration=0sec 2024-11-19T11:16:34,967 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:34,967 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5fb872772cd7220dc7447361f8e609a9:B 2024-11-19T11:16:34,967 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:34,967 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5fb872772cd7220dc7447361f8e609a9:A 2024-11-19T11:16:34,967 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:34,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742318_1494 (size=12301) 2024-11-19T11:16:34,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/d1cedb97a82148a287e5282d8baf411c 2024-11-19T11:16:34,968 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:34,968 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 5fb872772cd7220dc7447361f8e609a9/C is initiating minor compaction (all files) 2024-11-19T11:16:34,968 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5fb872772cd7220dc7447361f8e609a9/C in TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:34,968 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/b87b8d232ac74212bfe4e9788f9cc8c1, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/21d477e0461a4745863ff16453428389, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/8e2f5d662c3b44e5a2f6a104198efdb8] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp, totalSize=34.1 K 2024-11-19T11:16:34,968 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting b87b8d232ac74212bfe4e9788f9cc8c1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732014991233 2024-11-19T11:16:34,969 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 21d477e0461a4745863ff16453428389, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1732014992342 2024-11-19T11:16:34,969 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e2f5d662c3b44e5a2f6a104198efdb8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1732014993300 2024-11-19T11:16:34,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/3c8cbd45627d46c29e5d4797a68dc968 is 50, key is test_row_0/C:col10/1732014993504/Put/seqid=0 2024-11-19T11:16:34,974 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5fb872772cd7220dc7447361f8e609a9#C#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:34,974 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/ee3f474005c641dc96b2f0c77f4cc150 is 50, key is test_row_0/C:col10/1732014992347/Put/seqid=0 2024-11-19T11:16:34,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742320_1496 (size=12847) 2024-11-19T11:16:34,980 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/ee3f474005c641dc96b2f0c77f4cc150 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/ee3f474005c641dc96b2f0c77f4cc150 2024-11-19T11:16:34,983 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5fb872772cd7220dc7447361f8e609a9/C of 5fb872772cd7220dc7447361f8e609a9 into ee3f474005c641dc96b2f0c77f4cc150(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:34,983 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:34,983 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., storeName=5fb872772cd7220dc7447361f8e609a9/C, priority=13, startTime=1732014994544; duration=0sec 2024-11-19T11:16:34,983 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:34,983 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5fb872772cd7220dc7447361f8e609a9:C 2024-11-19T11:16:34,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742319_1495 (size=12301) 2024-11-19T11:16:35,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T11:16:35,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:35,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:35,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:35,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:35,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:35,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:35,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T11:16:35,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:35,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:35,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:35,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:35,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:35,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:35,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015055217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:35,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:35,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:35,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015055217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:35,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:35,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015055217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:35,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:35,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:35,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T11:16:35,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:35,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:35,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:35,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:35,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:35,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:35,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/3c8cbd45627d46c29e5d4797a68dc968 2024-11-19T11:16:35,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/962fe14ed14f4a62a25cac852198e41e as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/962fe14ed14f4a62a25cac852198e41e 2024-11-19T11:16:35,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/962fe14ed14f4a62a25cac852198e41e, entries=200, sequenceid=529, filesize=14.4 K 2024-11-19T11:16:35,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/d1cedb97a82148a287e5282d8baf411c as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/d1cedb97a82148a287e5282d8baf411c 2024-11-19T11:16:35,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/d1cedb97a82148a287e5282d8baf411c, entries=150, sequenceid=529, filesize=12.0 K 2024-11-19T11:16:35,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/3c8cbd45627d46c29e5d4797a68dc968 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/3c8cbd45627d46c29e5d4797a68dc968 2024-11-19T11:16:35,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/3c8cbd45627d46c29e5d4797a68dc968, entries=150, sequenceid=529, filesize=12.0 K 2024-11-19T11:16:35,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 5fb872772cd7220dc7447361f8e609a9 in 854ms, sequenceid=529, compaction requested=false 2024-11-19T11:16:35,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:35,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T11:16:35,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:35,482 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 5fb872772cd7220dc7447361f8e609a9 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-19T11:16:35,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=A 2024-11-19T11:16:35,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:35,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=B 2024-11-19T11:16:35,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:35,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=C 2024-11-19T11:16:35,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:35,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/a556168cabdc40dda55fae3b86a48027 is 50, key is test_row_0/A:col10/1732014994562/Put/seqid=0 2024-11-19T11:16:35,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742321_1497 (size=12301) 2024-11-19T11:16:35,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:35,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:35,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732014995701, max timestamp=1732014995727Num uniques 859; 2024-11-19T11:16:35,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T11:16:35,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015055744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:35,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:35,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015055745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:35,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:35,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015055747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:35,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:35,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015055748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:35,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:35,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015055749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:35,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T11:16:35,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015055858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:35,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015055867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:35,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:35,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015055867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:35,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:35,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015055867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:35,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:35,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:35,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015055877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:35,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:35,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:35,892 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/a556168cabdc40dda55fae3b86a48027 2024-11-19T11:16:35,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/531044ae68de42fabc702d8eaa2f2b2f is 50, key is test_row_0/B:col10/1732014994562/Put/seqid=0 2024-11-19T11:16:35,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742322_1498 (size=12301) 2024-11-19T11:16:36,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015056067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:36,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:36,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015056077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:36,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015056077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:36,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015056077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:36,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015056097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:36,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:36,302 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/531044ae68de42fabc702d8eaa2f2b2f 2024-11-19T11:16:36,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/648aa568df7a41a7b7b71a1e87c2057b is 50, key is test_row_0/C:col10/1732014994562/Put/seqid=0 2024-11-19T11:16:36,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742323_1499 (size=12301) 2024-11-19T11:16:36,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015056387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:36,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:36,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015056387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:36,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:36,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:36,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015056387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:36,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:36,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015056387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:36,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:36,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015056408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:36,711 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/648aa568df7a41a7b7b71a1e87c2057b 2024-11-19T11:16:36,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/a556168cabdc40dda55fae3b86a48027 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/a556168cabdc40dda55fae3b86a48027 2024-11-19T11:16:36,717 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/a556168cabdc40dda55fae3b86a48027, entries=150, sequenceid=541, filesize=12.0 K 2024-11-19T11:16:36,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/531044ae68de42fabc702d8eaa2f2b2f as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/531044ae68de42fabc702d8eaa2f2b2f 2024-11-19T11:16:36,720 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/531044ae68de42fabc702d8eaa2f2b2f, entries=150, sequenceid=541, filesize=12.0 K 2024-11-19T11:16:36,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/648aa568df7a41a7b7b71a1e87c2057b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/648aa568df7a41a7b7b71a1e87c2057b 2024-11-19T11:16:36,723 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/648aa568df7a41a7b7b71a1e87c2057b, entries=150, sequenceid=541, filesize=12.0 K 2024-11-19T11:16:36,723 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=138.65 KB/141980 for 5fb872772cd7220dc7447361f8e609a9 in 1241ms, sequenceid=541, compaction requested=true 2024-11-19T11:16:36,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:36,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:36,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-19T11:16:36,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-19T11:16:36,726 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-19T11:16:36,726 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0740 sec 2024-11-19T11:16:36,727 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 3.0780 sec 2024-11-19T11:16:36,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:36,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5fb872772cd7220dc7447361f8e609a9 3/3 column families, dataSize=145.36 KB heapSize=426.56 KB 2024-11-19T11:16:36,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=A 2024-11-19T11:16:36,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:36,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=B 2024-11-19T11:16:36,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:36,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=C 2024-11-19T11:16:36,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:36,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/25c816fe20504db8aa1bb7d03d89d592 is 50, key is test_row_0/A:col10/1732014995748/Put/seqid=0 2024-11-19T11:16:36,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742324_1500 (size=14741) 2024-11-19T11:16:36,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:36,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015056918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:36,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:36,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015056919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:36,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:36,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:36,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:36,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015056920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:36,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:36,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:36,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015056927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:36,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:36,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:36,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015056931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:36,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:36,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:37,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:37,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015057037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:37,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:37,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:37,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015057038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:37,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:37,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015057038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:37,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:37,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015057047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:37,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:37,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015057247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:37,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015057247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:37,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015057257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:37,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015057267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:37,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/25c816fe20504db8aa1bb7d03d89d592 2024-11-19T11:16:37,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/cb4c18b5e996449d9b3e7d001bdb062b is 50, key is test_row_0/B:col10/1732014995748/Put/seqid=0 2024-11-19T11:16:37,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742325_1501 (size=12301) 2024-11-19T11:16:37,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:37,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015057568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:37,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015057568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:37,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015057577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:37,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015057577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:37,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/cb4c18b5e996449d9b3e7d001bdb062b 2024-11-19T11:16:37,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/ae3a793915c94b35808153800155964b is 50, key is test_row_0/C:col10/1732014995748/Put/seqid=0 2024-11-19T11:16:37,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742326_1502 (size=12301) 2024-11-19T11:16:37,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/ae3a793915c94b35808153800155964b 2024-11-19T11:16:37,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/25c816fe20504db8aa1bb7d03d89d592 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/25c816fe20504db8aa1bb7d03d89d592 2024-11-19T11:16:37,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/25c816fe20504db8aa1bb7d03d89d592, entries=200, sequenceid=570, filesize=14.4 K 2024-11-19T11:16:37,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/cb4c18b5e996449d9b3e7d001bdb062b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/cb4c18b5e996449d9b3e7d001bdb062b 2024-11-19T11:16:37,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/cb4c18b5e996449d9b3e7d001bdb062b, entries=150, sequenceid=570, filesize=12.0 K 2024-11-19T11:16:37,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/ae3a793915c94b35808153800155964b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/ae3a793915c94b35808153800155964b 2024-11-19T11:16:37,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/ae3a793915c94b35808153800155964b, entries=150, sequenceid=570, filesize=12.0 K 2024-11-19T11:16:37,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~145.36 KB/148850, heapSize ~426.52 KB/436752, currentSize=33.54 KB/34350 for 5fb872772cd7220dc7447361f8e609a9 in 845ms, sequenceid=570, compaction requested=true 2024-11-19T11:16:37,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:37,760 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:16:37,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5fb872772cd7220dc7447361f8e609a9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:16:37,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:37,760 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:16:37,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5fb872772cd7220dc7447361f8e609a9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:16:37,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:37,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5fb872772cd7220dc7447361f8e609a9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:16:37,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:37,761 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54630 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:16:37,761 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:16:37,761 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 5fb872772cd7220dc7447361f8e609a9/A is initiating minor compaction (all files) 2024-11-19T11:16:37,761 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 5fb872772cd7220dc7447361f8e609a9/B is initiating minor compaction (all files) 2024-11-19T11:16:37,761 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5fb872772cd7220dc7447361f8e609a9/A in TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:37,761 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5fb872772cd7220dc7447361f8e609a9/B in TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:37,761 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/1fee37e04f96485a99cfe874e8151bec, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/d1cedb97a82148a287e5282d8baf411c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/531044ae68de42fabc702d8eaa2f2b2f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/cb4c18b5e996449d9b3e7d001bdb062b] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp, totalSize=48.6 K 2024-11-19T11:16:37,761 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/a9571c9bc1c548f383dd1103a38c58b7, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/962fe14ed14f4a62a25cac852198e41e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/a556168cabdc40dda55fae3b86a48027, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/25c816fe20504db8aa1bb7d03d89d592] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp, totalSize=53.3 K 2024-11-19T11:16:37,761 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9571c9bc1c548f383dd1103a38c58b7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1732014992347 2024-11-19T11:16:37,761 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fee37e04f96485a99cfe874e8151bec, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1732014992347 2024-11-19T11:16:37,762 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting d1cedb97a82148a287e5282d8baf411c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1732014993504 2024-11-19T11:16:37,762 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 962fe14ed14f4a62a25cac852198e41e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1732014993504 2024-11-19T11:16:37,762 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 531044ae68de42fabc702d8eaa2f2b2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732014994561 2024-11-19T11:16:37,762 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting a556168cabdc40dda55fae3b86a48027, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732014994561 2024-11-19T11:16:37,762 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting cb4c18b5e996449d9b3e7d001bdb062b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1732014995747 2024-11-19T11:16:37,762 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25c816fe20504db8aa1bb7d03d89d592, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1732014995743 2024-11-19T11:16:37,770 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5fb872772cd7220dc7447361f8e609a9#B#compaction#431 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:37,770 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/e3c41691d25d4115907bb082b691ab11 is 50, key is test_row_0/B:col10/1732014995748/Put/seqid=0 2024-11-19T11:16:37,772 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5fb872772cd7220dc7447361f8e609a9#A#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:37,773 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/0190785a9fd5448e8c005fadf792f982 is 50, key is test_row_0/A:col10/1732014995748/Put/seqid=0 2024-11-19T11:16:37,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742327_1503 (size=12983) 2024-11-19T11:16:37,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742328_1504 (size=12983) 2024-11-19T11:16:37,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T11:16:37,808 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:16:37,808 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:16:37,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-19T11:16:37,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T11:16:37,810 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:16:37,811 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:16:37,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:16:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T11:16:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:37,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5fb872772cd7220dc7447361f8e609a9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T11:16:37,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=A 2024-11-19T11:16:37,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:37,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=B 2024-11-19T11:16:37,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:37,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=C 2024-11-19T11:16:37,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:37,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/dfda701fe6384c5192cd435ac5912a5c is 50, key is test_row_0/A:col10/1732014997955/Put/seqid=0 2024-11-19T11:16:37,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T11:16:37,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742329_1505 (size=14741) 2024-11-19T11:16:37,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:37,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:37,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:37,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:37,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:37,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:38,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732014998023, max timestamp=1732014998092Num uniques 859; 2024-11-19T11:16:38,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T11:16:38,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732014998023, max timestamp=1732014998092Num uniques 859; 2024-11-19T11:16:38,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T11:16:38,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732014998023, max timestamp=1732014998092Num uniques 859; 2024-11-19T11:16:38,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T11:16:38,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T11:16:38,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:38,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:38,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:38,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:38,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:38,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:38,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T11:16:38,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:38,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015058121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:38,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015058122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:38,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:38,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015058125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:38,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015058126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:38,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:38,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015058127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:38,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:38,180 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/e3c41691d25d4115907bb082b691ab11 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/e3c41691d25d4115907bb082b691ab11 2024-11-19T11:16:38,181 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/0190785a9fd5448e8c005fadf792f982 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/0190785a9fd5448e8c005fadf792f982 2024-11-19T11:16:38,184 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5fb872772cd7220dc7447361f8e609a9/B of 5fb872772cd7220dc7447361f8e609a9 into e3c41691d25d4115907bb082b691ab11(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:38,184 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5fb872772cd7220dc7447361f8e609a9/A of 5fb872772cd7220dc7447361f8e609a9 into 0190785a9fd5448e8c005fadf792f982(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:38,184 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:38,184 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:38,184 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., storeName=5fb872772cd7220dc7447361f8e609a9/B, priority=12, startTime=1732014997760; duration=0sec 2024-11-19T11:16:38,184 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., storeName=5fb872772cd7220dc7447361f8e609a9/A, priority=12, startTime=1732014997760; duration=0sec 2024-11-19T11:16:38,184 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:38,185 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5fb872772cd7220dc7447361f8e609a9:B 2024-11-19T11:16:38,185 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:38,185 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5fb872772cd7220dc7447361f8e609a9:A 2024-11-19T11:16:38,185 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:16:38,186 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:16:38,186 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 5fb872772cd7220dc7447361f8e609a9/C is initiating minor compaction (all files) 2024-11-19T11:16:38,186 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5fb872772cd7220dc7447361f8e609a9/C in TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:38,186 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/ee3f474005c641dc96b2f0c77f4cc150, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/3c8cbd45627d46c29e5d4797a68dc968, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/648aa568df7a41a7b7b71a1e87c2057b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/ae3a793915c94b35808153800155964b] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp, totalSize=48.6 K 2024-11-19T11:16:38,186 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting ee3f474005c641dc96b2f0c77f4cc150, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1732014992347 2024-11-19T11:16:38,187 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c8cbd45627d46c29e5d4797a68dc968, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1732014993504 2024-11-19T11:16:38,187 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 648aa568df7a41a7b7b71a1e87c2057b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732014994561 2024-11-19T11:16:38,187 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting ae3a793915c94b35808153800155964b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1732014995747 2024-11-19T11:16:38,195 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5fb872772cd7220dc7447361f8e609a9#C#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:38,196 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/667c8eec435846cdb9e123667f1673dd is 50, key is test_row_0/C:col10/1732014995748/Put/seqid=0 2024-11-19T11:16:38,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742330_1506 (size=12983) 2024-11-19T11:16:38,209 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/667c8eec435846cdb9e123667f1673dd as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/667c8eec435846cdb9e123667f1673dd 2024-11-19T11:16:38,212 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5fb872772cd7220dc7447361f8e609a9/C of 5fb872772cd7220dc7447361f8e609a9 into 667c8eec435846cdb9e123667f1673dd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:38,212 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:38,212 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., storeName=5fb872772cd7220dc7447361f8e609a9/C, priority=12, startTime=1732014997760; duration=0sec 2024-11-19T11:16:38,212 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:38,212 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5fb872772cd7220dc7447361f8e609a9:C 2024-11-19T11:16:38,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:38,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015058238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:38,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:38,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:38,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015058238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:38,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:38,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:38,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015058247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:38,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:38,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015058247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:38,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:38,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:38,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:38,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015058247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:38,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:38,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T11:16:38,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:38,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:38,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:38,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:38,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:38,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:38,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/dfda701fe6384c5192cd435ac5912a5c 2024-11-19T11:16:38,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/9bf32042d5ae4cffbf1a3a99b65627eb is 50, key is test_row_0/B:col10/1732014997955/Put/seqid=0 2024-11-19T11:16:38,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742331_1507 (size=12301) 2024-11-19T11:16:38,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T11:16:38,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:38,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:38,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:38,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:38,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:38,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:38,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T11:16:38,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015058447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:38,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015058447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:38,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015058467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:38,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:38,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015058467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:38,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:38,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015058467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:38,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:38,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:38,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T11:16:38,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:38,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:38,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:38,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:38,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:38,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T11:16:38,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:38,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:38,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:38,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:38,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:38,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/9bf32042d5ae4cffbf1a3a99b65627eb 2024-11-19T11:16:38,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015058768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015058768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:38,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/4d2ba023016a423aa7fc9cd1ee980e41 is 50, key is test_row_0/C:col10/1732014997955/Put/seqid=0 2024-11-19T11:16:38,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742332_1508 (size=12301) 2024-11-19T11:16:38,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015058777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015058777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015058777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:38,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T11:16:38,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:38,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:38,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:38,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:38,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:38,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T11:16:39,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T11:16:39,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:39,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:39,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:39,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:39,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:39,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:39,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/4d2ba023016a423aa7fc9cd1ee980e41 2024-11-19T11:16:39,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T11:16:39,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:39,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:39,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:39,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:39,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:39,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/dfda701fe6384c5192cd435ac5912a5c as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/dfda701fe6384c5192cd435ac5912a5c 2024-11-19T11:16:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:39,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/dfda701fe6384c5192cd435ac5912a5c, entries=200, sequenceid=580, filesize=14.4 K 2024-11-19T11:16:39,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/9bf32042d5ae4cffbf1a3a99b65627eb as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/9bf32042d5ae4cffbf1a3a99b65627eb 2024-11-19T11:16:39,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/9bf32042d5ae4cffbf1a3a99b65627eb, entries=150, sequenceid=580, filesize=12.0 K 2024-11-19T11:16:39,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/4d2ba023016a423aa7fc9cd1ee980e41 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/4d2ba023016a423aa7fc9cd1ee980e41 2024-11-19T11:16:39,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/4d2ba023016a423aa7fc9cd1ee980e41, entries=150, sequenceid=580, filesize=12.0 K 2024-11-19T11:16:39,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 5fb872772cd7220dc7447361f8e609a9 in 1238ms, sequenceid=580, compaction requested=false 2024-11-19T11:16:39,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:39,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5fb872772cd7220dc7447361f8e609a9 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-19T11:16:39,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=A 2024-11-19T11:16:39,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:39,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=B 2024-11-19T11:16:39,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:39,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=C 2024-11-19T11:16:39,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:39,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/af9d2d8d06ef482b9c59063d2ba02db0 is 50, key is test_row_0/A:col10/1732014998126/Put/seqid=0 2024-11-19T11:16:39,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742333_1509 (size=14741) 2024-11-19T11:16:39,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=619 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/af9d2d8d06ef482b9c59063d2ba02db0 2024-11-19T11:16:39,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015059303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:39,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/9cdf97c7e7c94055a75419a350900353 is 50, key is test_row_0/B:col10/1732014998126/Put/seqid=0 2024-11-19T11:16:39,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742334_1510 (size=12301) 2024-11-19T11:16:39,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015059304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:39,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:39,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015059306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:39,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:39,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015059310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:39,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:39,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015059313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:39,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:39,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T11:16:39,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:39,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:39,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:39,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:39,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:39,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015059418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:39,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:39,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015059437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:39,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015059437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:39,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015059437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015059437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:39,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T11:16:39,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:39,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:39,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:39,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:39,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:39,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T11:16:39,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:39,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:39,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:39,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:39,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:39,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48522 deadline: 1732015059637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:39,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:39,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:39,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48588 deadline: 1732015059657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:39,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:39,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48608 deadline: 1732015059657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:39,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:39,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48542 deadline: 1732015059657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48562 deadline: 1732015059658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 , the old value is region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., hostname=16c88966c0af,35833,1732014855640, seqNum=2 because the exception is null or not the one we care about 2024-11-19T11:16:39,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=619 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/9cdf97c7e7c94055a75419a350900353 2024-11-19T11:16:39,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/fd63749ecedc44d2b80d1b57e899eea4 is 50, key is test_row_0/C:col10/1732014998126/Put/seqid=0 2024-11-19T11:16:39,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742335_1511 (size=12301) 2024-11-19T11:16:39,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=619 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/fd63749ecedc44d2b80d1b57e899eea4 2024-11-19T11:16:39,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/af9d2d8d06ef482b9c59063d2ba02db0 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/af9d2d8d06ef482b9c59063d2ba02db0 2024-11-19T11:16:39,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/af9d2d8d06ef482b9c59063d2ba02db0, entries=200, sequenceid=619, filesize=14.4 K 2024-11-19T11:16:39,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/9cdf97c7e7c94055a75419a350900353 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/9cdf97c7e7c94055a75419a350900353 2024-11-19T11:16:39,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/9cdf97c7e7c94055a75419a350900353, entries=150, sequenceid=619, filesize=12.0 K 2024-11-19T11:16:39,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/fd63749ecedc44d2b80d1b57e899eea4 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/fd63749ecedc44d2b80d1b57e899eea4 2024-11-19T11:16:39,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/fd63749ecedc44d2b80d1b57e899eea4, entries=150, sequenceid=619, filesize=12.0 K 2024-11-19T11:16:39,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for 5fb872772cd7220dc7447361f8e609a9 in 453ms, sequenceid=619, compaction requested=true 2024-11-19T11:16:39,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:39,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5fb872772cd7220dc7447361f8e609a9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:16:39,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:39,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5fb872772cd7220dc7447361f8e609a9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:16:39,751 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:39,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:39,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5fb872772cd7220dc7447361f8e609a9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:16:39,751 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:39,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:39,754 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:39,754 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 5fb872772cd7220dc7447361f8e609a9/A is initiating minor compaction (all files) 2024-11-19T11:16:39,754 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5fb872772cd7220dc7447361f8e609a9/A in TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:39,754 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/0190785a9fd5448e8c005fadf792f982, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/dfda701fe6384c5192cd435ac5912a5c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/af9d2d8d06ef482b9c59063d2ba02db0] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp, totalSize=41.5 K 2024-11-19T11:16:39,754 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:39,754 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 5fb872772cd7220dc7447361f8e609a9/B is initiating minor compaction (all files) 2024-11-19T11:16:39,754 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5fb872772cd7220dc7447361f8e609a9/B in TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:39,754 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/e3c41691d25d4115907bb082b691ab11, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/9bf32042d5ae4cffbf1a3a99b65627eb, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/9cdf97c7e7c94055a75419a350900353] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp, totalSize=36.7 K 2024-11-19T11:16:39,754 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0190785a9fd5448e8c005fadf792f982, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1732014995747 2024-11-19T11:16:39,755 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting e3c41691d25d4115907bb082b691ab11, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1732014995747 2024-11-19T11:16:39,755 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfda701fe6384c5192cd435ac5912a5c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1732014996916 2024-11-19T11:16:39,755 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bf32042d5ae4cffbf1a3a99b65627eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1732014996916 2024-11-19T11:16:39,755 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting af9d2d8d06ef482b9c59063d2ba02db0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=619, earliestPutTs=1732014998120 2024-11-19T11:16:39,755 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cdf97c7e7c94055a75419a350900353, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=619, earliestPutTs=1732014998125 2024-11-19T11:16:39,766 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5fb872772cd7220dc7447361f8e609a9#A#compaction#440 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:39,766 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/6cc704995f1c445faf58e271353bb81a is 50, key is test_row_0/A:col10/1732014998126/Put/seqid=0 2024-11-19T11:16:39,770 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5fb872772cd7220dc7447361f8e609a9#B#compaction#441 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:39,770 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/218a6e8cbdaf4a0cace9a4ad9ef1878c is 50, key is test_row_0/B:col10/1732014998126/Put/seqid=0 2024-11-19T11:16:39,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742336_1512 (size=13085) 2024-11-19T11:16:39,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T11:16:39,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:39,795 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 5fb872772cd7220dc7447361f8e609a9 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-19T11:16:39,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=A 2024-11-19T11:16:39,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:39,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=B 2024-11-19T11:16:39,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:39,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=C 2024-11-19T11:16:39,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:39,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/6cd8dfecd7264efe8e6d5079d642e755 is 50, key is test_row_0/A:col10/1732014999311/Put/seqid=0 2024-11-19T11:16:39,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742337_1513 (size=13085) 2024-11-19T11:16:39,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742338_1514 (size=12301) 2024-11-19T11:16:39,816 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=628 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/6cd8dfecd7264efe8e6d5079d642e755 2024-11-19T11:16:39,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/fc07eb1599f845fd8774c8ca6fa912ba is 50, key is test_row_0/B:col10/1732014999311/Put/seqid=0 2024-11-19T11:16:39,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742339_1515 (size=12301) 2024-11-19T11:16:39,833 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=628 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/fc07eb1599f845fd8774c8ca6fa912ba 2024-11-19T11:16:39,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/be2d773b67854e989ada1a45afccea1b is 50, key is test_row_0/C:col10/1732014999311/Put/seqid=0 2024-11-19T11:16:39,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742340_1516 (size=12301) 2024-11-19T11:16:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T11:16:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:39,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. as already flushing 2024-11-19T11:16:39,998 INFO [Thread-1993 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1993. 2024-11-19T11:16:39,998 DEBUG [Thread-1993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:16:39,998 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:39,998 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:16:39,998 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:39,999 INFO [Thread-1995 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1995. 2024-11-19T11:16:39,999 DEBUG [Thread-1995 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:16:39,999 DEBUG [Thread-1995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:39,999 DEBUG [Thread-1995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:39,999 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:16:39,999 INFO [Thread-2000 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2000. 2024-11-19T11:16:39,999 DEBUG [Thread-2000 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:16:39,999 DEBUG [Thread-2000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:39,999 DEBUG [Thread-2000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:39,999 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:16:40,000 INFO [Thread-1987 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1987. 2024-11-19T11:16:40,000 DEBUG [Thread-1987 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:16:40,000 DEBUG [Thread-1987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:40,000 DEBUG [Thread-1987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:40,001 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:16:40,001 INFO [Thread-1989 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1989. 2024-11-19T11:16:40,001 DEBUG [Thread-1989 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:16:40,001 DEBUG [Thread-1989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:40,001 DEBUG [Thread-1989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:40,001 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:16:40,001 INFO [Thread-2002 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2002. 2024-11-19T11:16:40,002 DEBUG [Thread-2002 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:16:40,002 DEBUG [Thread-2002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:40,002 DEBUG [Thread-2002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:40,002 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:16:40,006 INFO [Thread-1991 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1991. 2024-11-19T11:16:40,006 DEBUG [Thread-1991 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:16:40,006 DEBUG [Thread-1991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:40,006 DEBUG [Thread-1991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:40,006 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:16:40,007 INFO [Thread-1998 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1998. 2024-11-19T11:16:40,007 DEBUG [Thread-1998 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:16:40,007 DEBUG [Thread-1998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:40,007 DEBUG [Thread-1998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:40,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:16:40,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732014999983, max timestamp=1732015000005Num uniques 859; 2024-11-19T11:16:40,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T11:16:40,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732014999983, max timestamp=1732015000005Num uniques 859; 2024-11-19T11:16:40,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T11:16:40,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732014999983, max timestamp=1732015000005Num uniques 859; 2024-11-19T11:16:40,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T11:16:40,014 INFO [Thread-2004 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2004. 2024-11-19T11:16:40,014 DEBUG [Thread-2004 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:16:40,014 DEBUG [Thread-2004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:40,014 DEBUG [Thread-2004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:40,014 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:16:40,015 INFO [Thread-2006 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2006. 2024-11-19T11:16:40,015 DEBUG [Thread-2006 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:16:40,015 DEBUG [Thread-2006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:40,015 DEBUG [Thread-2006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:40,016 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:16:40,184 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/6cc704995f1c445faf58e271353bb81a as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/6cc704995f1c445faf58e271353bb81a 2024-11-19T11:16:40,189 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5fb872772cd7220dc7447361f8e609a9/A of 5fb872772cd7220dc7447361f8e609a9 into 6cc704995f1c445faf58e271353bb81a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:40,189 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:40,189 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., storeName=5fb872772cd7220dc7447361f8e609a9/A, priority=13, startTime=1732014999750; duration=0sec 2024-11-19T11:16:40,189 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:40,189 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5fb872772cd7220dc7447361f8e609a9:A 2024-11-19T11:16:40,189 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:40,190 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:40,190 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 5fb872772cd7220dc7447361f8e609a9/C is initiating minor compaction (all files) 2024-11-19T11:16:40,190 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5fb872772cd7220dc7447361f8e609a9/C in TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:40,190 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/667c8eec435846cdb9e123667f1673dd, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/4d2ba023016a423aa7fc9cd1ee980e41, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/fd63749ecedc44d2b80d1b57e899eea4] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp, totalSize=36.7 K 2024-11-19T11:16:40,191 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 667c8eec435846cdb9e123667f1673dd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1732014995747 2024-11-19T11:16:40,191 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d2ba023016a423aa7fc9cd1ee980e41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1732014996916 2024-11-19T11:16:40,191 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd63749ecedc44d2b80d1b57e899eea4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=619, earliestPutTs=1732014998125 2024-11-19T11:16:40,198 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5fb872772cd7220dc7447361f8e609a9#C#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:40,198 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/02f847f1c4114616becd45b57945e569 is 50, key is test_row_0/C:col10/1732014998126/Put/seqid=0 2024-11-19T11:16:40,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742341_1517 (size=13085) 2024-11-19T11:16:40,207 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/218a6e8cbdaf4a0cace9a4ad9ef1878c as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/218a6e8cbdaf4a0cace9a4ad9ef1878c 2024-11-19T11:16:40,210 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5fb872772cd7220dc7447361f8e609a9/B of 5fb872772cd7220dc7447361f8e609a9 into 218a6e8cbdaf4a0cace9a4ad9ef1878c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:40,210 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:40,210 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., storeName=5fb872772cd7220dc7447361f8e609a9/B, priority=13, startTime=1732014999750; duration=0sec 2024-11-19T11:16:40,210 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:40,210 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5fb872772cd7220dc7447361f8e609a9:B 2024-11-19T11:16:40,243 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=628 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/be2d773b67854e989ada1a45afccea1b 2024-11-19T11:16:40,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/6cd8dfecd7264efe8e6d5079d642e755 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/6cd8dfecd7264efe8e6d5079d642e755 2024-11-19T11:16:40,249 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/6cd8dfecd7264efe8e6d5079d642e755, entries=150, sequenceid=628, filesize=12.0 K 2024-11-19T11:16:40,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/fc07eb1599f845fd8774c8ca6fa912ba as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/fc07eb1599f845fd8774c8ca6fa912ba 2024-11-19T11:16:40,252 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/fc07eb1599f845fd8774c8ca6fa912ba, entries=150, sequenceid=628, filesize=12.0 K 2024-11-19T11:16:40,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/be2d773b67854e989ada1a45afccea1b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/be2d773b67854e989ada1a45afccea1b 2024-11-19T11:16:40,256 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/be2d773b67854e989ada1a45afccea1b, entries=150, sequenceid=628, filesize=12.0 K 2024-11-19T11:16:40,257 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=40.25 KB/41220 for 5fb872772cd7220dc7447361f8e609a9 in 462ms, sequenceid=628, compaction requested=false 2024-11-19T11:16:40,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:40,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:40,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-19T11:16:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-19T11:16:40,260 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-19T11:16:40,260 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4470 sec 2024-11-19T11:16:40,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.4510 sec 2024-11-19T11:16:40,613 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/02f847f1c4114616becd45b57945e569 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/02f847f1c4114616becd45b57945e569 2024-11-19T11:16:40,618 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5fb872772cd7220dc7447361f8e609a9/C of 5fb872772cd7220dc7447361f8e609a9 into 02f847f1c4114616becd45b57945e569(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:40,618 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5fb872772cd7220dc7447361f8e609a9: 2024-11-19T11:16:40,618 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., storeName=5fb872772cd7220dc7447361f8e609a9/C, priority=13, startTime=1732014999751; duration=0sec 2024-11-19T11:16:40,618 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:40,618 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5fb872772cd7220dc7447361f8e609a9:C 2024-11-19T11:16:41,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T11:16:41,969 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:16:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T11:16:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-19T11:16:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 245 2024-11-19T11:16:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 116 2024-11-19T11:16:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-19T11:16:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-19T11:16:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T11:16:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T11:16:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1106 2024-11-19T11:16:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3318 rows 2024-11-19T11:16:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1103 2024-11-19T11:16:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3305 rows 2024-11-19T11:16:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1107 2024-11-19T11:16:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3318 rows 2024-11-19T11:16:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1106 2024-11-19T11:16:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3316 rows 2024-11-19T11:16:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1106 2024-11-19T11:16:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3318 rows 2024-11-19T11:16:41,971 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T11:16:41,971 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T11:16:41,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:41,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:41,972 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:16:41,973 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T11:16:41,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T11:16:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T11:16:41,979 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732015001978"}]},"ts":"1732015001978"} 2024-11-19T11:16:41,981 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T11:16:41,981 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T11:16:41,982 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T11:16:41,984 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5fb872772cd7220dc7447361f8e609a9, UNASSIGN}] 2024-11-19T11:16:41,985 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5fb872772cd7220dc7447361f8e609a9, UNASSIGN 2024-11-19T11:16:41,986 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=5fb872772cd7220dc7447361f8e609a9, regionState=CLOSING, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:16:41,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5fb872772cd7220dc7447361f8e609a9, UNASSIGN because future has completed 2024-11-19T11:16:41,987 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T11:16:41,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640}] 2024-11-19T11:16:42,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T11:16:42,140 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:42,140 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T11:16:42,140 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing 5fb872772cd7220dc7447361f8e609a9, disabling compactions & flushes 2024-11-19T11:16:42,140 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:42,140 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:42,140 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. after waiting 0 ms 2024-11-19T11:16:42,140 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:42,140 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(2902): Flushing 5fb872772cd7220dc7447361f8e609a9 3/3 column families, dataSize=40.25 KB heapSize=96.70 KB 2024-11-19T11:16:42,141 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=A 2024-11-19T11:16:42,141 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:42,141 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=B 2024-11-19T11:16:42,141 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:42,141 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5fb872772cd7220dc7447361f8e609a9, store=C 2024-11-19T11:16:42,141 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:42,144 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/9005771cd4204fb98cc3e04e1f69d78e is 50, key is test_row_0/A:col10/1732015000005/Put/seqid=0 2024-11-19T11:16:42,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742342_1518 (size=12301) 2024-11-19T11:16:42,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T11:16:42,549 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=652 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/9005771cd4204fb98cc3e04e1f69d78e 2024-11-19T11:16:42,606 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/e6136d61e21d4140b5fe6aff554fa8b0 is 50, key is test_row_0/B:col10/1732015000005/Put/seqid=0 2024-11-19T11:16:42,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T11:16:42,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742343_1519 (size=12301) 2024-11-19T11:16:43,013 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=652 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/e6136d61e21d4140b5fe6aff554fa8b0 2024-11-19T11:16:43,025 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/8408b5a0e0604dd3961061e0fa1154dc is 50, key is test_row_0/C:col10/1732015000005/Put/seqid=0 2024-11-19T11:16:43,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742344_1520 (size=12301) 2024-11-19T11:16:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T11:16:43,433 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=652 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/8408b5a0e0604dd3961061e0fa1154dc 2024-11-19T11:16:43,442 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/A/9005771cd4204fb98cc3e04e1f69d78e as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/9005771cd4204fb98cc3e04e1f69d78e 2024-11-19T11:16:43,445 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/9005771cd4204fb98cc3e04e1f69d78e, entries=150, sequenceid=652, filesize=12.0 K 2024-11-19T11:16:43,446 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/B/e6136d61e21d4140b5fe6aff554fa8b0 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/e6136d61e21d4140b5fe6aff554fa8b0 2024-11-19T11:16:43,450 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/e6136d61e21d4140b5fe6aff554fa8b0, entries=150, sequenceid=652, filesize=12.0 K 2024-11-19T11:16:43,451 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/.tmp/C/8408b5a0e0604dd3961061e0fa1154dc as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/8408b5a0e0604dd3961061e0fa1154dc 2024-11-19T11:16:43,455 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/8408b5a0e0604dd3961061e0fa1154dc, entries=150, sequenceid=652, filesize=12.0 K 2024-11-19T11:16:43,456 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~96.66 KB/98976, currentSize=0 B/0 for 5fb872772cd7220dc7447361f8e609a9 in 1315ms, sequenceid=652, compaction requested=true 2024-11-19T11:16:43,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/a67d2c6cad094fb5b0d0a00ca394c147, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/aa1b63e2c48a41d58652b8d79411fd67, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/bca6061135764191a2a4244b1a06a261, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/0f5660a155c547cdba4ce9327a4d5b71, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/a00d483157a2401ab388ecd1e4883665, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/eaef461635e44087a3faa7c6dae4fbd4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/518bd5664b9140b281dc42a3e1b2520a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/425f830e2dbd4b7b8c0440efce6b3ddd, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/ceff646a36ea47229c49d83c22b2d98f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/363bc246da8e46f087fd4710e25357f8, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/78c50df82a414006a2395e8a484331c0, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/4a4fd6aa21ef455f88304fcee7299be4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/e07b7409f0ec4b248e3c033738e83d6e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/87fea6ec65a04bd59c707713234b2cfa, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/3f75c14ab85b4eb9a7991c9dc29ac112, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/a9571c9bc1c548f383dd1103a38c58b7, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/17c545cf58cf465c9db263a8d33e0d92, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/962fe14ed14f4a62a25cac852198e41e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/a556168cabdc40dda55fae3b86a48027, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/25c816fe20504db8aa1bb7d03d89d592, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/0190785a9fd5448e8c005fadf792f982, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/dfda701fe6384c5192cd435ac5912a5c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/af9d2d8d06ef482b9c59063d2ba02db0] to archive 2024-11-19T11:16:43,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T11:16:43,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/a67d2c6cad094fb5b0d0a00ca394c147 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/a67d2c6cad094fb5b0d0a00ca394c147 2024-11-19T11:16:43,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/aa1b63e2c48a41d58652b8d79411fd67 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/aa1b63e2c48a41d58652b8d79411fd67 2024-11-19T11:16:43,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/bca6061135764191a2a4244b1a06a261 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/bca6061135764191a2a4244b1a06a261 2024-11-19T11:16:43,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/0f5660a155c547cdba4ce9327a4d5b71 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/0f5660a155c547cdba4ce9327a4d5b71 2024-11-19T11:16:43,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/a00d483157a2401ab388ecd1e4883665 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/a00d483157a2401ab388ecd1e4883665 2024-11-19T11:16:43,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/eaef461635e44087a3faa7c6dae4fbd4 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/eaef461635e44087a3faa7c6dae4fbd4 2024-11-19T11:16:43,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/518bd5664b9140b281dc42a3e1b2520a to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/518bd5664b9140b281dc42a3e1b2520a 2024-11-19T11:16:43,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/425f830e2dbd4b7b8c0440efce6b3ddd to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/425f830e2dbd4b7b8c0440efce6b3ddd 2024-11-19T11:16:43,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/ceff646a36ea47229c49d83c22b2d98f to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/ceff646a36ea47229c49d83c22b2d98f 2024-11-19T11:16:43,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/363bc246da8e46f087fd4710e25357f8 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/363bc246da8e46f087fd4710e25357f8 2024-11-19T11:16:43,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/78c50df82a414006a2395e8a484331c0 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/78c50df82a414006a2395e8a484331c0 2024-11-19T11:16:43,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/4a4fd6aa21ef455f88304fcee7299be4 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/4a4fd6aa21ef455f88304fcee7299be4 2024-11-19T11:16:43,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/e07b7409f0ec4b248e3c033738e83d6e to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/e07b7409f0ec4b248e3c033738e83d6e 2024-11-19T11:16:43,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/87fea6ec65a04bd59c707713234b2cfa to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/87fea6ec65a04bd59c707713234b2cfa 2024-11-19T11:16:43,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/3f75c14ab85b4eb9a7991c9dc29ac112 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/3f75c14ab85b4eb9a7991c9dc29ac112 2024-11-19T11:16:43,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/a9571c9bc1c548f383dd1103a38c58b7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/a9571c9bc1c548f383dd1103a38c58b7 2024-11-19T11:16:43,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/17c545cf58cf465c9db263a8d33e0d92 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/17c545cf58cf465c9db263a8d33e0d92 2024-11-19T11:16:43,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/962fe14ed14f4a62a25cac852198e41e to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/962fe14ed14f4a62a25cac852198e41e 2024-11-19T11:16:43,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/a556168cabdc40dda55fae3b86a48027 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/a556168cabdc40dda55fae3b86a48027 2024-11-19T11:16:43,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/25c816fe20504db8aa1bb7d03d89d592 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/25c816fe20504db8aa1bb7d03d89d592 2024-11-19T11:16:43,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/0190785a9fd5448e8c005fadf792f982 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/0190785a9fd5448e8c005fadf792f982 2024-11-19T11:16:43,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/dfda701fe6384c5192cd435ac5912a5c to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/dfda701fe6384c5192cd435ac5912a5c 2024-11-19T11:16:43,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/af9d2d8d06ef482b9c59063d2ba02db0 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/af9d2d8d06ef482b9c59063d2ba02db0 2024-11-19T11:16:43,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/bd8d18e33ba040d9ac13d31d7a4b10cb, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/d5bf016fa74e4562a118442aedcf00fa, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/2e67b88139a544e987fdef351477c579, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/945498d6754d4dcba098d016a6d8a077, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/4ec2a229563e4527ad5a8e3181860563, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/825a87f285da40059917c03989ce150b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/3dbc8bd52f9b4a8fac1fa3cc5258be1a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/8443849a373d4228bb032383ff6ca6c0, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/801a824189c84853b1fa7ada12c58029, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/fe04fc4f34a846e9a219414aaceea98b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/28333748a34e4e9b9300684ee64dae5f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/05532c51a2fc423d9e14daa3f43f70ca, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/9ac0e7970bac49548cf04a08875a22c5, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/520f0804f22d4264b3adc7970223cb10, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/b8f7356f3e3b400081dc11d24614d3df, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/1fee37e04f96485a99cfe874e8151bec, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/7522d8e8ba5c47bdae2501c375863a57, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/d1cedb97a82148a287e5282d8baf411c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/531044ae68de42fabc702d8eaa2f2b2f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/e3c41691d25d4115907bb082b691ab11, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/cb4c18b5e996449d9b3e7d001bdb062b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/9bf32042d5ae4cffbf1a3a99b65627eb, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/9cdf97c7e7c94055a75419a350900353] to archive 2024-11-19T11:16:43,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T11:16:43,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/bd8d18e33ba040d9ac13d31d7a4b10cb to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/bd8d18e33ba040d9ac13d31d7a4b10cb 2024-11-19T11:16:43,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/d5bf016fa74e4562a118442aedcf00fa to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/d5bf016fa74e4562a118442aedcf00fa 2024-11-19T11:16:43,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/2e67b88139a544e987fdef351477c579 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/2e67b88139a544e987fdef351477c579 2024-11-19T11:16:43,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/945498d6754d4dcba098d016a6d8a077 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/945498d6754d4dcba098d016a6d8a077 2024-11-19T11:16:43,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/4ec2a229563e4527ad5a8e3181860563 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/4ec2a229563e4527ad5a8e3181860563 2024-11-19T11:16:43,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/825a87f285da40059917c03989ce150b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/825a87f285da40059917c03989ce150b 2024-11-19T11:16:43,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/3dbc8bd52f9b4a8fac1fa3cc5258be1a to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/3dbc8bd52f9b4a8fac1fa3cc5258be1a 2024-11-19T11:16:43,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/8443849a373d4228bb032383ff6ca6c0 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/8443849a373d4228bb032383ff6ca6c0 2024-11-19T11:16:43,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/801a824189c84853b1fa7ada12c58029 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/801a824189c84853b1fa7ada12c58029 2024-11-19T11:16:43,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/fe04fc4f34a846e9a219414aaceea98b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/fe04fc4f34a846e9a219414aaceea98b 2024-11-19T11:16:43,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/28333748a34e4e9b9300684ee64dae5f to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/28333748a34e4e9b9300684ee64dae5f 2024-11-19T11:16:43,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/05532c51a2fc423d9e14daa3f43f70ca to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/05532c51a2fc423d9e14daa3f43f70ca 2024-11-19T11:16:43,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/9ac0e7970bac49548cf04a08875a22c5 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/9ac0e7970bac49548cf04a08875a22c5 2024-11-19T11:16:43,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/520f0804f22d4264b3adc7970223cb10 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/520f0804f22d4264b3adc7970223cb10 2024-11-19T11:16:43,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/b8f7356f3e3b400081dc11d24614d3df to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/b8f7356f3e3b400081dc11d24614d3df 2024-11-19T11:16:43,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/1fee37e04f96485a99cfe874e8151bec to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/1fee37e04f96485a99cfe874e8151bec 2024-11-19T11:16:43,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/7522d8e8ba5c47bdae2501c375863a57 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/7522d8e8ba5c47bdae2501c375863a57 2024-11-19T11:16:43,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/d1cedb97a82148a287e5282d8baf411c to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/d1cedb97a82148a287e5282d8baf411c 2024-11-19T11:16:43,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/531044ae68de42fabc702d8eaa2f2b2f to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/531044ae68de42fabc702d8eaa2f2b2f 2024-11-19T11:16:43,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/e3c41691d25d4115907bb082b691ab11 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/e3c41691d25d4115907bb082b691ab11 2024-11-19T11:16:43,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/cb4c18b5e996449d9b3e7d001bdb062b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/cb4c18b5e996449d9b3e7d001bdb062b 2024-11-19T11:16:43,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/9bf32042d5ae4cffbf1a3a99b65627eb to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/9bf32042d5ae4cffbf1a3a99b65627eb 2024-11-19T11:16:43,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/9cdf97c7e7c94055a75419a350900353 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/9cdf97c7e7c94055a75419a350900353 2024-11-19T11:16:43,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/5084288f6735469d9678a3830c8e68e4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/268b721b842649fba22d492ab1f37e59, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/3fbb106d923d41b49f64d6874616a7e2, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/7fe670452e9148528b3b40fa329c10ea, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/9b9e5b4e4479459fa4f7b5648941766f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/2acc03361f7f4162a7e020cc3246ba1f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/d4b1de313acf4d8bbb42278017486783, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/679982c3ab52463d9158c30a730bb048, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/8bfce21705724dbcacae2426edccb506, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/cd95ca8be94a4021a95fd585b069e774, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/94efb2e1f5494d2a8ae664205fbbe5c5, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/66f501ea995548af8421d8a3fe0ae7f2, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/b87b8d232ac74212bfe4e9788f9cc8c1, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/9be08b2786284c03bcc3b2d4aa4f2c76, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/21d477e0461a4745863ff16453428389, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/ee3f474005c641dc96b2f0c77f4cc150, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/8e2f5d662c3b44e5a2f6a104198efdb8, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/3c8cbd45627d46c29e5d4797a68dc968, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/648aa568df7a41a7b7b71a1e87c2057b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/667c8eec435846cdb9e123667f1673dd, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/ae3a793915c94b35808153800155964b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/4d2ba023016a423aa7fc9cd1ee980e41, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/fd63749ecedc44d2b80d1b57e899eea4] to archive 2024-11-19T11:16:43,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T11:16:43,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/5084288f6735469d9678a3830c8e68e4 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/5084288f6735469d9678a3830c8e68e4 2024-11-19T11:16:43,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/268b721b842649fba22d492ab1f37e59 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/268b721b842649fba22d492ab1f37e59 2024-11-19T11:16:43,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/3fbb106d923d41b49f64d6874616a7e2 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/3fbb106d923d41b49f64d6874616a7e2 2024-11-19T11:16:43,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/7fe670452e9148528b3b40fa329c10ea to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/7fe670452e9148528b3b40fa329c10ea 2024-11-19T11:16:43,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/9b9e5b4e4479459fa4f7b5648941766f to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/9b9e5b4e4479459fa4f7b5648941766f 2024-11-19T11:16:43,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/2acc03361f7f4162a7e020cc3246ba1f to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/2acc03361f7f4162a7e020cc3246ba1f 2024-11-19T11:16:43,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/d4b1de313acf4d8bbb42278017486783 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/d4b1de313acf4d8bbb42278017486783 2024-11-19T11:16:43,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/679982c3ab52463d9158c30a730bb048 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/679982c3ab52463d9158c30a730bb048 2024-11-19T11:16:43,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/8bfce21705724dbcacae2426edccb506 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/8bfce21705724dbcacae2426edccb506 2024-11-19T11:16:43,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/cd95ca8be94a4021a95fd585b069e774 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/cd95ca8be94a4021a95fd585b069e774 2024-11-19T11:16:43,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/94efb2e1f5494d2a8ae664205fbbe5c5 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/94efb2e1f5494d2a8ae664205fbbe5c5 2024-11-19T11:16:43,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/66f501ea995548af8421d8a3fe0ae7f2 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/66f501ea995548af8421d8a3fe0ae7f2 2024-11-19T11:16:43,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/b87b8d232ac74212bfe4e9788f9cc8c1 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/b87b8d232ac74212bfe4e9788f9cc8c1 2024-11-19T11:16:43,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/9be08b2786284c03bcc3b2d4aa4f2c76 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/9be08b2786284c03bcc3b2d4aa4f2c76 2024-11-19T11:16:43,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/21d477e0461a4745863ff16453428389 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/21d477e0461a4745863ff16453428389 2024-11-19T11:16:43,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/ee3f474005c641dc96b2f0c77f4cc150 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/ee3f474005c641dc96b2f0c77f4cc150 2024-11-19T11:16:43,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/8e2f5d662c3b44e5a2f6a104198efdb8 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/8e2f5d662c3b44e5a2f6a104198efdb8 2024-11-19T11:16:43,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/3c8cbd45627d46c29e5d4797a68dc968 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/3c8cbd45627d46c29e5d4797a68dc968 2024-11-19T11:16:43,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/648aa568df7a41a7b7b71a1e87c2057b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/648aa568df7a41a7b7b71a1e87c2057b 2024-11-19T11:16:43,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/667c8eec435846cdb9e123667f1673dd to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/667c8eec435846cdb9e123667f1673dd 2024-11-19T11:16:43,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/ae3a793915c94b35808153800155964b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/ae3a793915c94b35808153800155964b 2024-11-19T11:16:43,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/4d2ba023016a423aa7fc9cd1ee980e41 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/4d2ba023016a423aa7fc9cd1ee980e41 2024-11-19T11:16:43,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/fd63749ecedc44d2b80d1b57e899eea4 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/fd63749ecedc44d2b80d1b57e899eea4 2024-11-19T11:16:43,515 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/recovered.edits/655.seqid, newMaxSeqId=655, maxSeqId=1 2024-11-19T11:16:43,515 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. 2024-11-19T11:16:43,516 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for 5fb872772cd7220dc7447361f8e609a9: Waiting for close lock at 1732015002140Running coprocessor pre-close hooks at 1732015002140Disabling compacts and flushes for region at 1732015002140Disabling writes for close at 1732015002140Obtaining lock to block concurrent updates at 1732015002140Preparing flush snapshotting stores in 5fb872772cd7220dc7447361f8e609a9 at 1732015002140Finished memstore snapshotting TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9., syncing WAL and waiting on mvcc, flushsize=dataSize=41220, getHeapSize=98976, getOffHeapSize=0, getCellsCount=900 at 1732015002141 (+1 ms)Flushing stores of TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9. at 1732015002141Flushing 5fb872772cd7220dc7447361f8e609a9/A: creating writer at 1732015002141Flushing 5fb872772cd7220dc7447361f8e609a9/A: appending metadata at 1732015002143 (+2 ms)Flushing 5fb872772cd7220dc7447361f8e609a9/A: closing flushed file at 1732015002143Flushing 5fb872772cd7220dc7447361f8e609a9/B: creating writer at 1732015002600 (+457 ms)Flushing 5fb872772cd7220dc7447361f8e609a9/B: appending metadata at 1732015002605 (+5 ms)Flushing 5fb872772cd7220dc7447361f8e609a9/B: closing flushed file at 1732015002605Flushing 5fb872772cd7220dc7447361f8e609a9/C: creating writer at 1732015003019 (+414 ms)Flushing 5fb872772cd7220dc7447361f8e609a9/C: appending metadata at 1732015003024 (+5 ms)Flushing 5fb872772cd7220dc7447361f8e609a9/C: closing flushed file at 1732015003024Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@723f7114: reopening flushed file at 1732015003441 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fac931a: reopening flushed file at 1732015003445 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63f68359: reopening flushed file at 1732015003450 (+5 ms)Finished flush of dataSize ~40.25 KB/41220, heapSize ~96.66 KB/98976, currentSize=0 B/0 for 5fb872772cd7220dc7447361f8e609a9 in 1315ms, sequenceid=652, compaction requested=true at 1732015003456 (+6 ms)Writing region close event to WAL at 1732015003512 (+56 ms)Running coprocessor post-close hooks at 1732015003515 (+3 ms)Closed at 1732015003515 2024-11-19T11:16:43,517 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed 5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:43,518 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=5fb872772cd7220dc7447361f8e609a9, regionState=CLOSED 2024-11-19T11:16:43,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 because future has completed 2024-11-19T11:16:43,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-19T11:16:43,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure 5fb872772cd7220dc7447361f8e609a9, server=16c88966c0af,35833,1732014855640 in 1.5330 sec 2024-11-19T11:16:43,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-19T11:16:43,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5fb872772cd7220dc7447361f8e609a9, UNASSIGN in 1.5370 sec 2024-11-19T11:16:43,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-19T11:16:43,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5410 sec 2024-11-19T11:16:43,526 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732015003526"}]},"ts":"1732015003526"} 2024-11-19T11:16:43,527 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T11:16:43,527 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T11:16:43,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5540 sec 2024-11-19T11:16:43,567 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T11:16:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T11:16:44,129 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:16:44,131 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T11:16:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:16:44,135 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:16:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T11:16:44,137 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:16:44,140 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:44,143 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A, FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B, FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C, FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/recovered.edits] 2024-11-19T11:16:44,146 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/6cc704995f1c445faf58e271353bb81a to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/6cc704995f1c445faf58e271353bb81a 2024-11-19T11:16:44,148 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/6cd8dfecd7264efe8e6d5079d642e755 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/6cd8dfecd7264efe8e6d5079d642e755 2024-11-19T11:16:44,149 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/9005771cd4204fb98cc3e04e1f69d78e to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/A/9005771cd4204fb98cc3e04e1f69d78e 2024-11-19T11:16:44,153 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/218a6e8cbdaf4a0cace9a4ad9ef1878c to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/218a6e8cbdaf4a0cace9a4ad9ef1878c 2024-11-19T11:16:44,154 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/e6136d61e21d4140b5fe6aff554fa8b0 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/e6136d61e21d4140b5fe6aff554fa8b0 2024-11-19T11:16:44,156 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/fc07eb1599f845fd8774c8ca6fa912ba to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/B/fc07eb1599f845fd8774c8ca6fa912ba 2024-11-19T11:16:44,159 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/02f847f1c4114616becd45b57945e569 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/02f847f1c4114616becd45b57945e569 2024-11-19T11:16:44,161 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/8408b5a0e0604dd3961061e0fa1154dc to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/8408b5a0e0604dd3961061e0fa1154dc 2024-11-19T11:16:44,163 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/be2d773b67854e989ada1a45afccea1b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/C/be2d773b67854e989ada1a45afccea1b 2024-11-19T11:16:44,167 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/recovered.edits/655.seqid to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9/recovered.edits/655.seqid 2024-11-19T11:16:44,168 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/5fb872772cd7220dc7447361f8e609a9 2024-11-19T11:16:44,168 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T11:16:44,172 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:16:44,175 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T11:16:44,179 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T11:16:44,181 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:16:44,181 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T11:16:44,181 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732015004181"}]},"ts":"9223372036854775807"} 2024-11-19T11:16:44,184 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T11:16:44,184 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5fb872772cd7220dc7447361f8e609a9, NAME => 'TestAcidGuarantees,,1732014977795.5fb872772cd7220dc7447361f8e609a9.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T11:16:44,184 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T11:16:44,184 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732015004184"}]},"ts":"9223372036854775807"} 2024-11-19T11:16:44,187 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T11:16:44,188 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:16:44,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 57 msec 2024-11-19T11:16:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T11:16:44,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T11:16:44,252 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:16:44,263 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251 (was 252), OpenFileDescriptor=453 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=423 (was 414) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11677 (was 11682) 2024-11-19T11:16:44,270 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=423, ProcessCount=11, AvailableMemoryMB=11676 2024-11-19T11:16:44,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T11:16:44,272 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:16:44,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T11:16:44,273 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T11:16:44,273 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:44,273 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-11-19T11:16:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-19T11:16:44,274 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T11:16:44,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742345_1521 (size=1061) 2024-11-19T11:16:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-19T11:16:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-19T11:16:44,681 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094 2024-11-19T11:16:44,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742346_1522 (size=53) 2024-11-19T11:16:44,759 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T11:16:44,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-19T11:16:45,088 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:16:45,088 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1a9e9054addbd8bfe6392d1e9f0ace31, disabling compactions & flushes 2024-11-19T11:16:45,088 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:45,088 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:45,088 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. after waiting 0 ms 2024-11-19T11:16:45,088 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:45,088 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:45,088 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1a9e9054addbd8bfe6392d1e9f0ace31: Waiting for close lock at 1732015005088Disabling compacts and flushes for region at 1732015005088Disabling writes for close at 1732015005088Writing region close event to WAL at 1732015005088Closed at 1732015005088 2024-11-19T11:16:45,089 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T11:16:45,089 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732015005089"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732015005089"}]},"ts":"1732015005089"} 2024-11-19T11:16:45,090 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T11:16:45,091 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T11:16:45,091 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732015005091"}]},"ts":"1732015005091"} 2024-11-19T11:16:45,092 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T11:16:45,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a9e9054addbd8bfe6392d1e9f0ace31, ASSIGN}] 2024-11-19T11:16:45,094 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a9e9054addbd8bfe6392d1e9f0ace31, ASSIGN 2024-11-19T11:16:45,094 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a9e9054addbd8bfe6392d1e9f0ace31, ASSIGN; state=OFFLINE, location=16c88966c0af,35833,1732014855640; forceNewPlan=false, retain=false 2024-11-19T11:16:45,245 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=1a9e9054addbd8bfe6392d1e9f0ace31, regionState=OPENING, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:16:45,247 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a9e9054addbd8bfe6392d1e9f0ace31, ASSIGN because future has completed 2024-11-19T11:16:45,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640}] 2024-11-19T11:16:45,405 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:45,405 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} 2024-11-19T11:16:45,406 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:45,406 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:16:45,406 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:45,406 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:45,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-19T11:16:45,409 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:45,411 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:16:45,412 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a9e9054addbd8bfe6392d1e9f0ace31 columnFamilyName A 2024-11-19T11:16:45,412 DEBUG [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:45,412 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] regionserver.HStore(327): Store=1a9e9054addbd8bfe6392d1e9f0ace31/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:16:45,412 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:45,413 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:16:45,413 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a9e9054addbd8bfe6392d1e9f0ace31 columnFamilyName B 2024-11-19T11:16:45,413 DEBUG [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:45,414 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] regionserver.HStore(327): Store=1a9e9054addbd8bfe6392d1e9f0ace31/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:16:45,414 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:45,414 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:16:45,415 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a9e9054addbd8bfe6392d1e9f0ace31 columnFamilyName C 2024-11-19T11:16:45,415 DEBUG [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:45,415 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] regionserver.HStore(327): Store=1a9e9054addbd8bfe6392d1e9f0ace31/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:16:45,415 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:45,415 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:45,416 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:45,416 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:45,417 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:45,417 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:45,417 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T11:16:45,418 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:45,419 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T11:16:45,419 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened 1a9e9054addbd8bfe6392d1e9f0ace31; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64215956, jitterRate=-0.043107688426971436}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T11:16:45,419 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:45,420 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for 1a9e9054addbd8bfe6392d1e9f0ace31: Running coprocessor pre-open hook at 1732015005406Writing region info on filesystem at 1732015005406Initializing all the Stores at 1732015005408 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732015005408Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732015005408Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732015005408Cleaning up temporary data from old regions at 1732015005417 (+9 ms)Running coprocessor post-open hooks at 1732015005419 (+2 ms)Region opened successfully at 1732015005420 (+1 ms) 2024-11-19T11:16:45,420 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., pid=140, masterSystemTime=1732015005399 2024-11-19T11:16:45,422 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:45,422 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:45,422 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=1a9e9054addbd8bfe6392d1e9f0ace31, regionState=OPEN, openSeqNum=2, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:16:45,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 because future has completed 2024-11-19T11:16:45,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-19T11:16:45,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 in 177 msec 2024-11-19T11:16:45,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-19T11:16:45,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a9e9054addbd8bfe6392d1e9f0ace31, ASSIGN in 333 msec 2024-11-19T11:16:45,428 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T11:16:45,429 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732015005428"}]},"ts":"1732015005428"} 2024-11-19T11:16:45,430 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T11:16:45,430 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T11:16:45,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1580 sec 2024-11-19T11:16:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-19T11:16:46,418 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:16:46,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@348042de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:46,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:16:46,420 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:16:46,421 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:16:46,421 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:16:46,421 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:16:46,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2307099d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:46,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:16:46,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:16:46,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:46,423 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:16:46,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18dd7406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:46,424 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:16:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:16:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:16:46,425 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:16:46,426 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T11:16:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 16c88966c0af,41441,1732014854424 2024-11-19T11:16:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@aa13fd5 2024-11-19T11:16:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T11:16:46,427 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T11:16:46,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T11:16:46,428 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:16:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T11:16:46,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742347_1523 (size=1097) 2024-11-19T11:16:46,837 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T11:16:46,838 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T11:16:46,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T11:16:46,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a9e9054addbd8bfe6392d1e9f0ace31, REOPEN/MOVE}] 2024-11-19T11:16:46,843 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a9e9054addbd8bfe6392d1e9f0ace31, REOPEN/MOVE 2024-11-19T11:16:46,843 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=1a9e9054addbd8bfe6392d1e9f0ace31, regionState=CLOSING, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:16:46,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a9e9054addbd8bfe6392d1e9f0ace31, REOPEN/MOVE because future has completed 2024-11-19T11:16:46,845 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T11:16:46,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640}] 2024-11-19T11:16:46,998 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:46,998 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T11:16:46,998 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 1a9e9054addbd8bfe6392d1e9f0ace31, disabling compactions & flushes 2024-11-19T11:16:46,998 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:46,999 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:46,999 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. after waiting 0 ms 2024-11-19T11:16:46,999 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:47,008 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T11:16:47,010 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:47,010 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 1a9e9054addbd8bfe6392d1e9f0ace31: Waiting for close lock at 1732015006998Running coprocessor pre-close hooks at 1732015006998Disabling compacts and flushes for region at 1732015006998Disabling writes for close at 1732015006999 (+1 ms)Writing region close event to WAL at 1732015007000 (+1 ms)Running coprocessor post-close hooks at 1732015007010 (+10 ms)Closed at 1732015007010 2024-11-19T11:16:47,010 WARN [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: 1a9e9054addbd8bfe6392d1e9f0ace31 to self. 2024-11-19T11:16:47,012 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:47,012 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=1a9e9054addbd8bfe6392d1e9f0ace31, regionState=CLOSED 2024-11-19T11:16:47,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 because future has completed 2024-11-19T11:16:47,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-19T11:16:47,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 in 169 msec 2024-11-19T11:16:47,017 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a9e9054addbd8bfe6392d1e9f0ace31, REOPEN/MOVE; state=CLOSED, location=16c88966c0af,35833,1732014855640; forceNewPlan=false, retain=true 2024-11-19T11:16:47,168 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=1a9e9054addbd8bfe6392d1e9f0ace31, regionState=OPENING, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:16:47,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a9e9054addbd8bfe6392d1e9f0ace31, REOPEN/MOVE because future has completed 2024-11-19T11:16:47,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640}] 2024-11-19T11:16:47,328 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:47,328 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} 2024-11-19T11:16:47,328 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:47,329 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:16:47,329 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:47,329 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:47,330 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:47,330 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:16:47,330 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a9e9054addbd8bfe6392d1e9f0ace31 columnFamilyName A 2024-11-19T11:16:47,331 DEBUG [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:47,331 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] regionserver.HStore(327): Store=1a9e9054addbd8bfe6392d1e9f0ace31/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:16:47,332 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:47,332 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:16:47,332 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a9e9054addbd8bfe6392d1e9f0ace31 columnFamilyName B 2024-11-19T11:16:47,332 DEBUG [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:47,333 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] regionserver.HStore(327): Store=1a9e9054addbd8bfe6392d1e9f0ace31/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:16:47,333 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:47,333 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T11:16:47,333 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a9e9054addbd8bfe6392d1e9f0ace31 columnFamilyName C 2024-11-19T11:16:47,333 DEBUG [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:47,334 INFO [StoreOpener-1a9e9054addbd8bfe6392d1e9f0ace31-1 {}] regionserver.HStore(327): Store=1a9e9054addbd8bfe6392d1e9f0ace31/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:16:47,334 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:47,334 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:47,334 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:47,335 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:47,336 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:47,336 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:47,336 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T11:16:47,338 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:47,338 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened 1a9e9054addbd8bfe6392d1e9f0ace31; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65524799, jitterRate=-0.023604407906532288}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T11:16:47,339 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:47,339 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for 1a9e9054addbd8bfe6392d1e9f0ace31: Running coprocessor pre-open hook at 1732015007329Writing region info on filesystem at 1732015007329Initializing all the Stores at 1732015007329Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732015007329Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732015007329Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732015007329Cleaning up temporary data from old regions at 1732015007336 (+7 ms)Running coprocessor post-open hooks at 1732015007339 (+3 ms)Region opened successfully at 1732015007339 2024-11-19T11:16:47,339 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., pid=145, masterSystemTime=1732015007326 2024-11-19T11:16:47,341 DEBUG [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:47,341 INFO [RS_OPEN_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:47,341 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=1a9e9054addbd8bfe6392d1e9f0ace31, regionState=OPEN, openSeqNum=5, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:16:47,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 because future has completed 2024-11-19T11:16:47,344 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-11-19T11:16:47,344 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 in 171 msec 2024-11-19T11:16:47,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-19T11:16:47,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a9e9054addbd8bfe6392d1e9f0ace31, REOPEN/MOVE in 502 msec 2024-11-19T11:16:47,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-19T11:16:47,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 506 msec 2024-11-19T11:16:47,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 920 msec 2024-11-19T11:16:47,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T11:16:47,350 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:16:47,351 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24560e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,351 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:16:47,351 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:16:47,351 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:16:47,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:16:47,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:16:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b9e9bb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:16:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:16:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:47,352 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:16:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8872eb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,353 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,353 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:16:47,353 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:16:47,354 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:16:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:16:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:16:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f7af9b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:16:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:16:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:47,355 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:16:47,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31e8673f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,356 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,356 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:16:47,356 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:16:47,356 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:16:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:16:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:16:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21ff94ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:16:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:16:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:47,357 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:16:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bfe246c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,358 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,358 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:16:47,358 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:16:47,358 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:16:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:16:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:16:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c7ba4f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:16:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:16:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:47,359 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:16:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27dd21b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41947491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,360 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:16:47,360 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:16:47,361 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:16:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:16:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:16:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@673d2afc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:16:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:16:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:47,362 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:16:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ee22bbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bc8537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,362 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:16:47,363 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:16:47,363 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:16:47,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:16:47,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:16:47,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a6818da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:16:47,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:16:47,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:47,364 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35076, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:16:47,364 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e5d2f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,365 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7595d14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,365 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:16:47,365 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:16:47,366 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:16:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:16:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:16:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74cce6e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:16:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:16:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:47,367 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:16:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f01f93b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,368 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf79e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,368 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:16:47,368 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:16:47,368 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:16:47,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:16:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:16:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68fc755f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:16:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:16:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:47,369 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:16:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55eec1b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d0e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,370 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:16:47,370 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:16:47,371 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:16:47,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:16:47,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:16:47,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@698ccdb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:16:47,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:16:47,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:47,372 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:16:47,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e1361c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9a2803, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,373 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16c88966c0af,41441,-1 for getting cluster id 2024-11-19T11:16:47,373 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:16:47,374 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b4a9e94-f011-4c02-bbc9-3820ab93d5c2' 2024-11-19T11:16:47,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:16:47,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b4a9e94-f011-4c02-bbc9-3820ab93d5c2" 2024-11-19T11:16:47,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@403a308c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16c88966c0af,41441,-1] 2024-11-19T11:16:47,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:16:47,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:16:47,375 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:16:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ce01be5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T11:16:47,376 DEBUG [Thread-2351 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:16:47,376 DEBUG [Thread-2357 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:16:47,376 DEBUG [Thread-2355 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:16:47,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:16:47,376 DEBUG [Thread-2348 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:16:47,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:16:47,377 DEBUG [Thread-2340 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:16:47,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:16:47,377 DEBUG [Thread-2342 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:16:47,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:16:47,377 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:16:47,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:16:47,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:16:47,378 DEBUG [Thread-2346 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:16:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:16:47,378 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:16:47,378 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:16:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:16:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:16:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:16:47,378 DEBUG [Thread-2359 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:16:47,378 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:16:47,378 DEBUG [Thread-2344 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:16:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-19T11:16:47,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:16:47,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:16:47,379 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:16:47,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:16:47,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:16:47,379 DEBUG [Thread-2353 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:16:47,379 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:16:47,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:16:47,379 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:16:47,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:16:47,380 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:16:47,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:16:47,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T11:16:47,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:16:47,380 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:16:47,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:16:47,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:16:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16c88966c0af,35833,1732014855640, seqNum=-1] 2024-11-19T11:16:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:16:47,381 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:16:47,382 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:16:47,382 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:16:47,382 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:16:47,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:16:47,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:16:47,383 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:16:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:16:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:16:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:16:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:16:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:16:47,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5] 2024-11-19T11:16:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:47,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a9e9054addbd8bfe6392d1e9f0ace31 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T11:16:47,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=A 2024-11-19T11:16:47,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:47,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=B 2024-11-19T11:16:47,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:47,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=C 2024-11-19T11:16:47,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:47,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:47,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015067406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:47,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:47,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732015007405, max timestamp=1732015007409Num uniques 859; 2024-11-19T11:16:47,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:47,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732015007405, max timestamp=1732015007409Num uniques 859; 2024-11-19T11:16:47,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:47,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732015007405, max timestamp=1732015007409Num uniques 859; 2024-11-19T11:16:47,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16c88966c0af:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T11:16:47,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ab42bc2051424f0aaa4c1a12209fd0ff_1a9e9054addbd8bfe6392d1e9f0ace31 is 50, key is test_row_0/A:col10/1732015007388/Put/seqid=0 2024-11-19T11:16:47,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742348_1524 (size=9714) 2024-11-19T11:16:47,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:47,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55098 deadline: 1732015067427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:47,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015067428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:47,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015067429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55080 deadline: 1732015067430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T11:16:47,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015067517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:47,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T11:16:47,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:47,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:47,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:47,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:47,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:47,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55098 deadline: 1732015067538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015067538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:47,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015067538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55080 deadline: 1732015067538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:47,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T11:16:47,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:47,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:47,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:47,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:47,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T11:16:47,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015067727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:47,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55098 deadline: 1732015067748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:47,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:47,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015067748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:47,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015067748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:47,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:47,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55080 deadline: 1732015067748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:47,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:47,826 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:47,829 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ab42bc2051424f0aaa4c1a12209fd0ff_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ab42bc2051424f0aaa4c1a12209fd0ff_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:47,830 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/6f5da12a56fb49d78c27ca07dce419bb, store: [table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:16:47,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/6f5da12a56fb49d78c27ca07dce419bb is 175, key is test_row_0/A:col10/1732015007388/Put/seqid=0 2024-11-19T11:16:47,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742349_1525 (size=22361) 2024-11-19T11:16:47,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T11:16:47,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:47,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:47,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:47,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:47,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:47,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T11:16:47,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:47,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:47,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:47,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:47,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:48,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T11:16:48,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:48,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015068037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:48,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:48,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:48,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:48,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:48,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55098 deadline: 1732015068058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:48,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:48,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015068058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:48,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:48,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015068058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:48,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:48,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55080 deadline: 1732015068058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:48,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T11:16:48,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:48,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:48,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:48,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:48,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:48,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:48,233 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/6f5da12a56fb49d78c27ca07dce419bb 2024-11-19T11:16:48,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/24a2b8982fb34cbcb5334dafcce15ca7 is 50, key is test_row_0/B:col10/1732015007388/Put/seqid=0 2024-11-19T11:16:48,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742350_1526 (size=9657) 2024-11-19T11:16:48,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T11:16:48,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:48,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:48,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:48,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:48,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:48,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:48,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T11:16:48,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:48,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:48,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:48,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:48,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:48,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T11:16:48,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015068548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:48,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:48,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:48,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:48,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:48,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55098 deadline: 1732015068567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015068567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:48,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:48,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55080 deadline: 1732015068567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015068567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:48,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T11:16:48,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:48,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:48,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:48,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:48,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:48,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/24a2b8982fb34cbcb5334dafcce15ca7 2024-11-19T11:16:48,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/654c49f10ec6404087109a8cb2883d4c is 50, key is test_row_0/C:col10/1732015007388/Put/seqid=0 2024-11-19T11:16:48,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742351_1527 (size=9657) 2024-11-19T11:16:48,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T11:16:48,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:48,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:48,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:48,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:48,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:48,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:48,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T11:16:48,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:48,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:48,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:48,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:48,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:49,038 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T11:16:49,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T11:16:49,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:49,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:49,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:49,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:49,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:49,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:49,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/654c49f10ec6404087109a8cb2883d4c 2024-11-19T11:16:49,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/6f5da12a56fb49d78c27ca07dce419bb as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/6f5da12a56fb49d78c27ca07dce419bb 2024-11-19T11:16:49,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/6f5da12a56fb49d78c27ca07dce419bb, entries=100, sequenceid=16, filesize=21.8 K 2024-11-19T11:16:49,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/24a2b8982fb34cbcb5334dafcce15ca7 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/24a2b8982fb34cbcb5334dafcce15ca7 2024-11-19T11:16:49,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/24a2b8982fb34cbcb5334dafcce15ca7, entries=100, sequenceid=16, filesize=9.4 K 2024-11-19T11:16:49,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/654c49f10ec6404087109a8cb2883d4c as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/654c49f10ec6404087109a8cb2883d4c 2024-11-19T11:16:49,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/654c49f10ec6404087109a8cb2883d4c, entries=100, sequenceid=16, filesize=9.4 K 2024-11-19T11:16:49,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 1a9e9054addbd8bfe6392d1e9f0ace31 in 1710ms, sequenceid=16, compaction requested=false 2024-11-19T11:16:49,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:16:49,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T11:16:49,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:49,220 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 1a9e9054addbd8bfe6392d1e9f0ace31 3/3 column families, dataSize=120.76 KB heapSize=345.14 KB 2024-11-19T11:16:49,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=A 2024-11-19T11:16:49,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:49,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=B 2024-11-19T11:16:49,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:49,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=C 2024-11-19T11:16:49,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T11:16:49,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b358cfd3b81e438e952930cab86b0eb7_1a9e9054addbd8bfe6392d1e9f0ace31 is 50, key is test_row_0/A:col10/1732015007429/Put/seqid=0 2024-11-19T11:16:49,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742352_1528 (size=12154) 2024-11-19T11:16:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T11:16:49,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:49,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015069585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:49,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:49,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:49,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:49,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015069586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:49,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:49,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015069587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55098 deadline: 1732015069587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:49,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55080 deadline: 1732015069587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:49,591 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_0' in 1a9e9054addbd8bfe6392d1e9f0ace31 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:49,592 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_1' in 1a9e9054addbd8bfe6392d1e9f0ace31 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:49,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:49,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:49,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:49,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:49,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:49,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:49,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:49,632 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b358cfd3b81e438e952930cab86b0eb7_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b358cfd3b81e438e952930cab86b0eb7_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:49,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/99b034b73888449483edb9b5a677eb60, store: [table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:16:49,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/99b034b73888449483edb9b5a677eb60 is 175, key is test_row_0/A:col10/1732015007429/Put/seqid=0 2024-11-19T11:16:49,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742353_1529 (size=30955) 2024-11-19T11:16:49,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015069697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:49,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:49,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:49,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:49,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015069697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:49,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015069697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:49,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:49,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:49,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:49,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:49,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:49,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:49,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:49,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015069907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:49,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:49,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:49,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:49,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015069907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:49,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:49,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:49,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:49,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015069907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:49,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:49,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:49,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:50,036 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/99b034b73888449483edb9b5a677eb60 2024-11-19T11:16:50,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/26ec59464367472f8d0f4fb938965885 is 50, key is test_row_0/B:col10/1732015007429/Put/seqid=0 2024-11-19T11:16:50,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742354_1530 (size=12001) 2024-11-19T11:16:50,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:50,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015070218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:50,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:50,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:50,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:50,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:50,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015070218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:50,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:50,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:50,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:50,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:50,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015070218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:50,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:50,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:50,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:50,446 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/26ec59464367472f8d0f4fb938965885 2024-11-19T11:16:50,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/64ab895b3c724a2cbe931da876bf3aea is 50, key is test_row_0/C:col10/1732015007429/Put/seqid=0 2024-11-19T11:16:50,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742355_1531 (size=12001) 2024-11-19T11:16:50,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015070727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:50,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:50,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:50,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:50,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015070727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:50,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015070727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:50,854 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/64ab895b3c724a2cbe931da876bf3aea 2024-11-19T11:16:50,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/99b034b73888449483edb9b5a677eb60 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/99b034b73888449483edb9b5a677eb60 2024-11-19T11:16:50,861 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/99b034b73888449483edb9b5a677eb60, entries=150, sequenceid=51, filesize=30.2 K 2024-11-19T11:16:50,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/26ec59464367472f8d0f4fb938965885 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/26ec59464367472f8d0f4fb938965885 2024-11-19T11:16:50,865 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/26ec59464367472f8d0f4fb938965885, entries=150, sequenceid=51, filesize=11.7 K 2024-11-19T11:16:50,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/64ab895b3c724a2cbe931da876bf3aea as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/64ab895b3c724a2cbe931da876bf3aea 2024-11-19T11:16:50,868 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/64ab895b3c724a2cbe931da876bf3aea, entries=150, sequenceid=51, filesize=11.7 K 2024-11-19T11:16:50,869 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~345.09 KB/353376, currentSize=60.38 KB/61830 for 1a9e9054addbd8bfe6392d1e9f0ace31 in 1650ms, sequenceid=51, compaction requested=false 2024-11-19T11:16:50,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:16:50,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:50,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-19T11:16:50,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-19T11:16:50,874 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-19T11:16:50,874 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4880 sec 2024-11-19T11:16:50,876 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 3.4980 sec 2024-11-19T11:16:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T11:16:51,548 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:16:51,548 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:16:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-19T11:16:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T11:16:51,549 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:16:51,550 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:16:51,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:16:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:51,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a9e9054addbd8bfe6392d1e9f0ace31 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T11:16:51,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=A 2024-11-19T11:16:51,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:51,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=B 2024-11-19T11:16:51,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:51,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=C 2024-11-19T11:16:51,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:51,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111986d53b4ff5f8438b924f5751b3eb2688_1a9e9054addbd8bfe6392d1e9f0ace31 is 50, key is test_row_0/A:col10/1732015009586/Put/seqid=0 2024-11-19T11:16:51,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742356_1532 (size=12154) 2024-11-19T11:16:51,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55098 deadline: 1732015071646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:51,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55080 deadline: 1732015071647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:51,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:51,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:51,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T11:16:51,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T11:16:51,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:51,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:51,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:51,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:51,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:51,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015071737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:51,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015071737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:51,739 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_1' in 1a9e9054addbd8bfe6392d1e9f0ace31 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:51,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:51,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:51,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:51,740 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_0' in 1a9e9054addbd8bfe6392d1e9f0ace31 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:51,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:51,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:51,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:51,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015071748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:51,750 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_0' in 1a9e9054addbd8bfe6392d1e9f0ace31 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:51,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55098 deadline: 1732015071757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:51,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:51,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:51,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55080 deadline: 1732015071757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:51,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:51,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:51,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:51,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T11:16:51,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:51,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:51,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:51,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:51,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T11:16:51,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55098 deadline: 1732015071967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:51,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:51,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55080 deadline: 1732015071967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:51,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:51,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:51,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:51,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:51,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:52,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T11:16:52,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:52,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:52,030 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:52,033 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111986d53b4ff5f8438b924f5751b3eb2688_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111986d53b4ff5f8438b924f5751b3eb2688_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:52,033 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/c771c2bfa6354d48b3649651888cf145, store: [table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:16:52,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/c771c2bfa6354d48b3649651888cf145 is 175, key is test_row_0/A:col10/1732015009586/Put/seqid=0 2024-11-19T11:16:52,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742357_1533 (size=30955) 2024-11-19T11:16:52,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T11:16:52,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:52,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:52,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:52,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:52,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T11:16:52,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55098 deadline: 1732015072278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:52,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:52,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55080 deadline: 1732015072278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:52,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:52,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:52,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:52,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:52,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:52,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T11:16:52,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:52,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:52,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:52,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:52,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:52,437 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/c771c2bfa6354d48b3649651888cf145 2024-11-19T11:16:52,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/5d90cb062f5141709e2b93668a548afb is 50, key is test_row_0/B:col10/1732015009586/Put/seqid=0 2024-11-19T11:16:52,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742358_1534 (size=12001) 2024-11-19T11:16:52,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T11:16:52,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:52,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:52,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:52,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:52,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:52,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T11:16:52,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:52,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:52,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:52,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:52,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T11:16:52,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T11:16:52,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:52,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:52,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:52,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:52,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:52,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55098 deadline: 1732015072788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:52,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:52,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:52,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:52,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55080 deadline: 1732015072788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:52,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:52,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:52,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:52,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/5d90cb062f5141709e2b93668a548afb 2024-11-19T11:16:52,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/575d658157b84e9da3f39bbe4bbb85cc is 50, key is test_row_0/C:col10/1732015009586/Put/seqid=0 2024-11-19T11:16:52,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742359_1535 (size=12001) 2024-11-19T11:16:52,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T11:16:52,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:52,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:52,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:52,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:52,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:53,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T11:16:53,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:53,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:53,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:53,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:53,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:53,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T11:16:53,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:53,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:53,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:53,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:53,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:53,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:53,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/575d658157b84e9da3f39bbe4bbb85cc 2024-11-19T11:16:53,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/c771c2bfa6354d48b3649651888cf145 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/c771c2bfa6354d48b3649651888cf145 2024-11-19T11:16:53,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/c771c2bfa6354d48b3649651888cf145, entries=150, sequenceid=64, filesize=30.2 K 2024-11-19T11:16:53,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/5d90cb062f5141709e2b93668a548afb as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/5d90cb062f5141709e2b93668a548afb 2024-11-19T11:16:53,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/5d90cb062f5141709e2b93668a548afb, entries=150, sequenceid=64, filesize=11.7 K 2024-11-19T11:16:53,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/575d658157b84e9da3f39bbe4bbb85cc as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/575d658157b84e9da3f39bbe4bbb85cc 2024-11-19T11:16:53,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/575d658157b84e9da3f39bbe4bbb85cc, entries=150, sequenceid=64, filesize=11.7 K 2024-11-19T11:16:53,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1a9e9054addbd8bfe6392d1e9f0ace31 in 1656ms, sequenceid=64, compaction requested=true 2024-11-19T11:16:53,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:16:53,266 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:53,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a9e9054addbd8bfe6392d1e9f0ace31:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:16:53,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:53,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a9e9054addbd8bfe6392d1e9f0ace31:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:16:53,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:53,266 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:53,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a9e9054addbd8bfe6392d1e9f0ace31:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:16:53,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:53,266 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 84271 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:53,266 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 1a9e9054addbd8bfe6392d1e9f0ace31/A is initiating minor compaction (all files) 2024-11-19T11:16:53,266 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a9e9054addbd8bfe6392d1e9f0ace31/A in TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:53,266 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:53,266 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 1a9e9054addbd8bfe6392d1e9f0ace31/B is initiating minor compaction (all files) 2024-11-19T11:16:53,266 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/6f5da12a56fb49d78c27ca07dce419bb, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/99b034b73888449483edb9b5a677eb60, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/c771c2bfa6354d48b3649651888cf145] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp, totalSize=82.3 K 2024-11-19T11:16:53,266 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a9e9054addbd8bfe6392d1e9f0ace31/B in TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:53,267 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/24a2b8982fb34cbcb5334dafcce15ca7, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/26ec59464367472f8d0f4fb938965885, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/5d90cb062f5141709e2b93668a548afb] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp, totalSize=32.9 K 2024-11-19T11:16:53,267 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:53,267 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. files: [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/6f5da12a56fb49d78c27ca07dce419bb, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/99b034b73888449483edb9b5a677eb60, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/c771c2bfa6354d48b3649651888cf145] 2024-11-19T11:16:53,267 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 24a2b8982fb34cbcb5334dafcce15ca7, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732015007388 2024-11-19T11:16:53,267 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f5da12a56fb49d78c27ca07dce419bb, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732015007388 2024-11-19T11:16:53,267 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 26ec59464367472f8d0f4fb938965885, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732015007422 2024-11-19T11:16:53,267 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99b034b73888449483edb9b5a677eb60, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732015007422 2024-11-19T11:16:53,267 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting c771c2bfa6354d48b3649651888cf145, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732015009584 2024-11-19T11:16:53,267 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d90cb062f5141709e2b93668a548afb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732015009584 2024-11-19T11:16:53,271 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:16:53,273 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a9e9054addbd8bfe6392d1e9f0ace31#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:53,273 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/d6350042017a47a4ac62b86bb4b430e3 is 50, key is test_row_0/B:col10/1732015009586/Put/seqid=0 2024-11-19T11:16:53,275 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119e500861ec91f457fb1c776cba6df488c_1a9e9054addbd8bfe6392d1e9f0ace31 store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:16:53,276 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119e500861ec91f457fb1c776cba6df488c_1a9e9054addbd8bfe6392d1e9f0ace31, store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:16:53,276 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e500861ec91f457fb1c776cba6df488c_1a9e9054addbd8bfe6392d1e9f0ace31 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:16:53,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742360_1536 (size=12104) 2024-11-19T11:16:53,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742361_1537 (size=4469) 2024-11-19T11:16:53,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T11:16:53,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:53,381 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 1a9e9054addbd8bfe6392d1e9f0ace31 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T11:16:53,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=A 2024-11-19T11:16:53,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:53,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=B 2024-11-19T11:16:53,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:53,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=C 2024-11-19T11:16:53,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:53,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111903f17c48103141e9a094dfc881df125c_1a9e9054addbd8bfe6392d1e9f0ace31 is 50, key is test_row_0/A:col10/1732015011640/Put/seqid=0 2024-11-19T11:16:53,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742362_1538 (size=12154) 2024-11-19T11:16:53,684 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/d6350042017a47a4ac62b86bb4b430e3 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/d6350042017a47a4ac62b86bb4b430e3 2024-11-19T11:16:53,687 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a9e9054addbd8bfe6392d1e9f0ace31/B of 1a9e9054addbd8bfe6392d1e9f0ace31 into d6350042017a47a4ac62b86bb4b430e3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:53,687 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:16:53,687 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., storeName=1a9e9054addbd8bfe6392d1e9f0ace31/B, priority=13, startTime=1732015013266; duration=0sec 2024-11-19T11:16:53,687 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:53,687 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a9e9054addbd8bfe6392d1e9f0ace31:B 2024-11-19T11:16:53,688 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:53,689 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:53,689 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 1a9e9054addbd8bfe6392d1e9f0ace31/C is initiating minor compaction (all files) 2024-11-19T11:16:53,689 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a9e9054addbd8bfe6392d1e9f0ace31/C in TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:53,689 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/654c49f10ec6404087109a8cb2883d4c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/64ab895b3c724a2cbe931da876bf3aea, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/575d658157b84e9da3f39bbe4bbb85cc] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp, totalSize=32.9 K 2024-11-19T11:16:53,689 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 654c49f10ec6404087109a8cb2883d4c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732015007388 2024-11-19T11:16:53,690 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 64ab895b3c724a2cbe931da876bf3aea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732015007422 2024-11-19T11:16:53,690 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a9e9054addbd8bfe6392d1e9f0ace31#A#compaction#458 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:53,691 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/b244dab4084e4e6e8178ea434e4fd88a is 175, key is test_row_0/A:col10/1732015009586/Put/seqid=0 2024-11-19T11:16:53,691 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 575d658157b84e9da3f39bbe4bbb85cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732015009584 2024-11-19T11:16:53,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742363_1539 (size=31058) 2024-11-19T11:16:53,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T11:16:53,702 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a9e9054addbd8bfe6392d1e9f0ace31#C#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:53,702 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/4945819ed81d4949beacc6687ea7542e is 50, key is test_row_0/C:col10/1732015009586/Put/seqid=0 2024-11-19T11:16:53,703 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/b244dab4084e4e6e8178ea434e4fd88a as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/b244dab4084e4e6e8178ea434e4fd88a 2024-11-19T11:16:53,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742364_1540 (size=12104) 2024-11-19T11:16:53,710 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a9e9054addbd8bfe6392d1e9f0ace31/A of 1a9e9054addbd8bfe6392d1e9f0ace31 into b244dab4084e4e6e8178ea434e4fd88a(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:53,710 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:16:53,710 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., storeName=1a9e9054addbd8bfe6392d1e9f0ace31/A, priority=13, startTime=1732015013265; duration=0sec 2024-11-19T11:16:53,710 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:53,710 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a9e9054addbd8bfe6392d1e9f0ace31:A 2024-11-19T11:16:53,713 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/4945819ed81d4949beacc6687ea7542e as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/4945819ed81d4949beacc6687ea7542e 2024-11-19T11:16:53,719 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a9e9054addbd8bfe6392d1e9f0ace31/C of 1a9e9054addbd8bfe6392d1e9f0ace31 into 4945819ed81d4949beacc6687ea7542e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:53,719 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:16:53,719 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., storeName=1a9e9054addbd8bfe6392d1e9f0ace31/C, priority=13, startTime=1732015013266; duration=0sec 2024-11-19T11:16:53,719 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:53,719 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a9e9054addbd8bfe6392d1e9f0ace31:C 2024-11-19T11:16:53,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:53,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015073775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:53,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:53,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:53,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:53,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:53,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015073776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:53,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:53,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:53,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:53,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015073778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:53,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:53,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:53,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:53,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:53,800 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111903f17c48103141e9a094dfc881df125c_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111903f17c48103141e9a094dfc881df125c_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:53,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/5a3f39b65a2742ec89a22ad6f812e95f, store: [table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:16:53,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:53,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55080 deadline: 1732015073797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:53,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/5a3f39b65a2742ec89a22ad6f812e95f is 175, key is test_row_0/A:col10/1732015011640/Put/seqid=0 2024-11-19T11:16:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55098 deadline: 1732015073797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:53,801 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_0' in 1a9e9054addbd8bfe6392d1e9f0ace31 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:53,801 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_2' in 1a9e9054addbd8bfe6392d1e9f0ace31 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:53,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742365_1541 (size=30955) 2024-11-19T11:16:53,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015073887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:53,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015073887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:53,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015073887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:54,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015074097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:54,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015074097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:54,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015074097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:54,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:54,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:54,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:54,204 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/5a3f39b65a2742ec89a22ad6f812e95f 2024-11-19T11:16:54,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/daf45d553f904bc7a579a54a90b6e382 is 50, key is test_row_0/B:col10/1732015011640/Put/seqid=0 2024-11-19T11:16:54,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742366_1542 (size=12001) 2024-11-19T11:16:54,213 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/daf45d553f904bc7a579a54a90b6e382 2024-11-19T11:16:54,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/8102e9060c6842fda0f0ca8e1e3ff7f8 is 50, key is test_row_0/C:col10/1732015011640/Put/seqid=0 2024-11-19T11:16:54,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742367_1543 (size=12001) 2024-11-19T11:16:54,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015074407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:54,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015074407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:54,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015074407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:54,622 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/8102e9060c6842fda0f0ca8e1e3ff7f8 2024-11-19T11:16:54,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/5a3f39b65a2742ec89a22ad6f812e95f as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/5a3f39b65a2742ec89a22ad6f812e95f 2024-11-19T11:16:54,628 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/5a3f39b65a2742ec89a22ad6f812e95f, entries=150, sequenceid=83, filesize=30.2 K 2024-11-19T11:16:54,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/daf45d553f904bc7a579a54a90b6e382 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/daf45d553f904bc7a579a54a90b6e382 2024-11-19T11:16:54,631 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/daf45d553f904bc7a579a54a90b6e382, entries=150, sequenceid=83, filesize=11.7 K 2024-11-19T11:16:54,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/8102e9060c6842fda0f0ca8e1e3ff7f8 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/8102e9060c6842fda0f0ca8e1e3ff7f8 2024-11-19T11:16:54,636 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/8102e9060c6842fda0f0ca8e1e3ff7f8, entries=150, sequenceid=83, filesize=11.7 K 2024-11-19T11:16:54,638 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1a9e9054addbd8bfe6392d1e9f0ace31 in 1256ms, sequenceid=83, compaction requested=false 2024-11-19T11:16:54,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:16:54,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:54,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-19T11:16:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-19T11:16:54,644 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-19T11:16:54,644 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0920 sec 2024-11-19T11:16:54,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 3.0960 sec 2024-11-19T11:16:54,759 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T11:16:54,759 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T11:16:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:54,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a9e9054addbd8bfe6392d1e9f0ace31 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T11:16:54,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=A 2024-11-19T11:16:54,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:54,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=B 2024-11-19T11:16:54,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:54,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=C 2024-11-19T11:16:54,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:54,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bce7284f434e41059c0b16e5aeae1508_1a9e9054addbd8bfe6392d1e9f0ace31 is 50, key is test_row_0/A:col10/1732015014920/Put/seqid=0 2024-11-19T11:16:54,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742368_1544 (size=17034) 2024-11-19T11:16:54,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015074939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:54,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015074941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:54,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015074942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:55,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015075047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:55,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015075047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:55,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015075057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:55,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:55,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:55,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:55,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:55,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015075258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:55,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:55,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:55,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:55,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:55,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015075258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:55,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:55,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:55,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:55,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015075267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:55,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:55,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:55,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:55,334 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:55,337 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bce7284f434e41059c0b16e5aeae1508_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bce7284f434e41059c0b16e5aeae1508_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:55,338 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/e6b8ef3793014551aa48edfe8d8eb182, store: [table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:16:55,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/e6b8ef3793014551aa48edfe8d8eb182 is 175, key is test_row_0/A:col10/1732015014920/Put/seqid=0 2024-11-19T11:16:55,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742369_1545 (size=48139) 2024-11-19T11:16:55,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015075568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:55,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:55,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:55,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:55,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015075568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:55,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015075577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:55,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:55,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:55,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T11:16:55,728 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:16:55,729 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:16:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-19T11:16:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T11:16:55,735 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:16:55,744 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:16:55,744 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:16:55,786 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=102, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/e6b8ef3793014551aa48edfe8d8eb182 2024-11-19T11:16:55,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/ca08e21157dd4502b390d7656970268a is 50, key is test_row_0/B:col10/1732015014920/Put/seqid=0 2024-11-19T11:16:55,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742370_1546 (size=12001) 2024-11-19T11:16:55,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55080 deadline: 1732015075807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:55,812 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_0' in 1a9e9054addbd8bfe6392d1e9f0ace31 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:55,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:55,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:55,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:55,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/ca08e21157dd4502b390d7656970268a 2024-11-19T11:16:55,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/24299380c38c41ed8cc2d6aede367cae is 50, key is test_row_0/C:col10/1732015014920/Put/seqid=0 2024-11-19T11:16:55,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55098 deadline: 1732015075827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:55,832 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_2' in 1a9e9054addbd8bfe6392d1e9f0ace31 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:55,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:55,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:55,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T11:16:55,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742371_1547 (size=12001) 2024-11-19T11:16:55,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T11:16:55,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:55,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:55,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:55,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:55,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T11:16:56,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T11:16:56,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:56,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:56,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:56,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:56,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:56,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015076077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:56,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015076077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:56,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:56,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:56,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015076088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:56,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:56,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:56,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:56,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T11:16:56,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:56,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:56,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:56,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:56,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:56,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/24299380c38c41ed8cc2d6aede367cae 2024-11-19T11:16:56,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/e6b8ef3793014551aa48edfe8d8eb182 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/e6b8ef3793014551aa48edfe8d8eb182 2024-11-19T11:16:56,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/e6b8ef3793014551aa48edfe8d8eb182, entries=250, sequenceid=102, filesize=47.0 K 2024-11-19T11:16:56,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/ca08e21157dd4502b390d7656970268a as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/ca08e21157dd4502b390d7656970268a 2024-11-19T11:16:56,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/ca08e21157dd4502b390d7656970268a, entries=150, sequenceid=102, filesize=11.7 K 2024-11-19T11:16:56,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/24299380c38c41ed8cc2d6aede367cae as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/24299380c38c41ed8cc2d6aede367cae 2024-11-19T11:16:56,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/24299380c38c41ed8cc2d6aede367cae, entries=150, sequenceid=102, filesize=11.7 K 2024-11-19T11:16:56,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 1a9e9054addbd8bfe6392d1e9f0ace31 in 1341ms, sequenceid=102, compaction requested=true 2024-11-19T11:16:56,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:16:56,261 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:56,262 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:56,262 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 1a9e9054addbd8bfe6392d1e9f0ace31/A is initiating minor compaction (all files) 2024-11-19T11:16:56,262 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a9e9054addbd8bfe6392d1e9f0ace31/A in TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:56,262 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/b244dab4084e4e6e8178ea434e4fd88a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/5a3f39b65a2742ec89a22ad6f812e95f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/e6b8ef3793014551aa48edfe8d8eb182] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp, totalSize=107.6 K 2024-11-19T11:16:56,262 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:56,262 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. files: [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/b244dab4084e4e6e8178ea434e4fd88a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/5a3f39b65a2742ec89a22ad6f812e95f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/e6b8ef3793014551aa48edfe8d8eb182] 2024-11-19T11:16:56,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a9e9054addbd8bfe6392d1e9f0ace31:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:16:56,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:56,263 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:56,263 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting b244dab4084e4e6e8178ea434e4fd88a, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732015009584 2024-11-19T11:16:56,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a9e9054addbd8bfe6392d1e9f0ace31:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:16:56,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:56,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a9e9054addbd8bfe6392d1e9f0ace31:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:16:56,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:56,264 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a3f39b65a2742ec89a22ad6f812e95f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732015011640 2024-11-19T11:16:56,265 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6b8ef3793014551aa48edfe8d8eb182, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732015013768 2024-11-19T11:16:56,265 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:56,265 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 1a9e9054addbd8bfe6392d1e9f0ace31/B is initiating minor compaction (all files) 2024-11-19T11:16:56,265 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a9e9054addbd8bfe6392d1e9f0ace31/B in TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:56,265 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/d6350042017a47a4ac62b86bb4b430e3, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/daf45d553f904bc7a579a54a90b6e382, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/ca08e21157dd4502b390d7656970268a] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp, totalSize=35.3 K 2024-11-19T11:16:56,266 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting d6350042017a47a4ac62b86bb4b430e3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732015009584 2024-11-19T11:16:56,267 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting daf45d553f904bc7a579a54a90b6e382, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732015011640 2024-11-19T11:16:56,267 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting ca08e21157dd4502b390d7656970268a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732015013768 2024-11-19T11:16:56,288 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:16:56,291 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a9e9054addbd8bfe6392d1e9f0ace31#B#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:56,291 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/1d10180dccc748fca1a75a6608389506 is 50, key is test_row_0/B:col10/1732015014920/Put/seqid=0 2024-11-19T11:16:56,301 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119e59bdf32ab164931a045833d3ded2456_1a9e9054addbd8bfe6392d1e9f0ace31 store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:16:56,303 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119e59bdf32ab164931a045833d3ded2456_1a9e9054addbd8bfe6392d1e9f0ace31, store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:16:56,303 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e59bdf32ab164931a045833d3ded2456_1a9e9054addbd8bfe6392d1e9f0ace31 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:16:56,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742372_1548 (size=12207) 2024-11-19T11:16:56,346 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/1d10180dccc748fca1a75a6608389506 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/1d10180dccc748fca1a75a6608389506 2024-11-19T11:16:56,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742373_1549 (size=4469) 2024-11-19T11:16:56,347 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a9e9054addbd8bfe6392d1e9f0ace31#A#compaction#467 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:56,348 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/5a6688d029cf49e4a04c98a137ce9c4c is 175, key is test_row_0/A:col10/1732015014920/Put/seqid=0 2024-11-19T11:16:56,351 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a9e9054addbd8bfe6392d1e9f0ace31/B of 1a9e9054addbd8bfe6392d1e9f0ace31 into 1d10180dccc748fca1a75a6608389506(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:56,351 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:16:56,351 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., storeName=1a9e9054addbd8bfe6392d1e9f0ace31/B, priority=13, startTime=1732015016263; duration=0sec 2024-11-19T11:16:56,351 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:56,351 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a9e9054addbd8bfe6392d1e9f0ace31:B 2024-11-19T11:16:56,351 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:56,353 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:56,353 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 1a9e9054addbd8bfe6392d1e9f0ace31/C is initiating minor compaction (all files) 2024-11-19T11:16:56,353 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a9e9054addbd8bfe6392d1e9f0ace31/C in TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:56,354 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/4945819ed81d4949beacc6687ea7542e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/8102e9060c6842fda0f0ca8e1e3ff7f8, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/24299380c38c41ed8cc2d6aede367cae] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp, totalSize=35.3 K 2024-11-19T11:16:56,354 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 4945819ed81d4949beacc6687ea7542e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732015009584 2024-11-19T11:16:56,354 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8102e9060c6842fda0f0ca8e1e3ff7f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732015011640 2024-11-19T11:16:56,354 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 24299380c38c41ed8cc2d6aede367cae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732015013768 2024-11-19T11:16:56,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T11:16:56,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:56,357 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 1a9e9054addbd8bfe6392d1e9f0ace31 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T11:16:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T11:16:56,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=A 2024-11-19T11:16:56,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742374_1550 (size=31161) 2024-11-19T11:16:56,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:56,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=B 2024-11-19T11:16:56,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:56,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=C 2024-11-19T11:16:56,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:56,370 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a9e9054addbd8bfe6392d1e9f0ace31#C#compaction#469 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:56,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ecafafaf1b9f4e28bd6530266caec5d5_1a9e9054addbd8bfe6392d1e9f0ace31 is 50, key is test_row_0/A:col10/1732015014941/Put/seqid=0 2024-11-19T11:16:56,371 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/b165fd7779e94f128883cdb7a84e806a is 50, key is test_row_0/C:col10/1732015014920/Put/seqid=0 2024-11-19T11:16:56,374 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/5a6688d029cf49e4a04c98a137ce9c4c as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/5a6688d029cf49e4a04c98a137ce9c4c 2024-11-19T11:16:56,381 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a9e9054addbd8bfe6392d1e9f0ace31/A of 1a9e9054addbd8bfe6392d1e9f0ace31 into 5a6688d029cf49e4a04c98a137ce9c4c(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:56,381 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:16:56,381 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., storeName=1a9e9054addbd8bfe6392d1e9f0ace31/A, priority=13, startTime=1732015016261; duration=0sec 2024-11-19T11:16:56,381 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:56,381 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a9e9054addbd8bfe6392d1e9f0ace31:A 2024-11-19T11:16:56,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742375_1551 (size=12207) 2024-11-19T11:16:56,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742376_1552 (size=12154) 2024-11-19T11:16:56,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:56,413 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/b165fd7779e94f128883cdb7a84e806a as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/b165fd7779e94f128883cdb7a84e806a 2024-11-19T11:16:56,414 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ecafafaf1b9f4e28bd6530266caec5d5_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ecafafaf1b9f4e28bd6530266caec5d5_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/c01dad77b5384c308d879bbae71880c5, store: [table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:16:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/c01dad77b5384c308d879bbae71880c5 is 175, key is test_row_0/A:col10/1732015014941/Put/seqid=0 2024-11-19T11:16:56,420 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a9e9054addbd8bfe6392d1e9f0ace31/C of 1a9e9054addbd8bfe6392d1e9f0ace31 into b165fd7779e94f128883cdb7a84e806a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:56,420 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:16:56,420 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., storeName=1a9e9054addbd8bfe6392d1e9f0ace31/C, priority=13, startTime=1732015016264; duration=0sec 2024-11-19T11:16:56,420 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:56,420 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a9e9054addbd8bfe6392d1e9f0ace31:C 2024-11-19T11:16:56,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742377_1553 (size=30955) 2024-11-19T11:16:56,462 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/c01dad77b5384c308d879bbae71880c5 2024-11-19T11:16:56,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/391a5cec20f746f3bd68b21efd396421 is 50, key is test_row_0/B:col10/1732015014941/Put/seqid=0 2024-11-19T11:16:56,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742378_1554 (size=12001) 2024-11-19T11:16:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T11:16:56,894 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/391a5cec20f746f3bd68b21efd396421 2024-11-19T11:16:56,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/03c8e20a43cd4ad9ab7364a9b828abe5 is 50, key is test_row_0/C:col10/1732015014941/Put/seqid=0 2024-11-19T11:16:56,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742379_1555 (size=12001) 2024-11-19T11:16:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:57,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:57,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015077124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:57,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:57,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:57,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015077132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015077133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:57,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:57,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:57,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015077237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:57,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:57,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015077247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015077248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:57,335 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/03c8e20a43cd4ad9ab7364a9b828abe5 2024-11-19T11:16:57,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/c01dad77b5384c308d879bbae71880c5 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/c01dad77b5384c308d879bbae71880c5 2024-11-19T11:16:57,355 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/c01dad77b5384c308d879bbae71880c5, entries=150, sequenceid=119, filesize=30.2 K 2024-11-19T11:16:57,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/391a5cec20f746f3bd68b21efd396421 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/391a5cec20f746f3bd68b21efd396421 2024-11-19T11:16:57,362 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/391a5cec20f746f3bd68b21efd396421, entries=150, sequenceid=119, filesize=11.7 K 2024-11-19T11:16:57,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/03c8e20a43cd4ad9ab7364a9b828abe5 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/03c8e20a43cd4ad9ab7364a9b828abe5 2024-11-19T11:16:57,368 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/03c8e20a43cd4ad9ab7364a9b828abe5, entries=150, sequenceid=119, filesize=11.7 K 2024-11-19T11:16:57,369 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 1a9e9054addbd8bfe6392d1e9f0ace31 in 1012ms, sequenceid=119, compaction requested=false 2024-11-19T11:16:57,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:16:57,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:57,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-19T11:16:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-19T11:16:57,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-19T11:16:57,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6270 sec 2024-11-19T11:16:57,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.6450 sec 2024-11-19T11:16:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:57,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a9e9054addbd8bfe6392d1e9f0ace31 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T11:16:57,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=A 2024-11-19T11:16:57,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:57,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=B 2024-11-19T11:16:57,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:57,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=C 2024-11-19T11:16:57,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:57,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194114cbb2839f45aeaef386a6a0b438a9_1a9e9054addbd8bfe6392d1e9f0ace31 is 50, key is test_row_0/A:col10/1732015017448/Put/seqid=0 2024-11-19T11:16:57,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742380_1556 (size=14794) 2024-11-19T11:16:57,464 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:57,467 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194114cbb2839f45aeaef386a6a0b438a9_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194114cbb2839f45aeaef386a6a0b438a9_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:57,470 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/4dd44284c1e64894966c7086f84bfe20, store: [table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:16:57,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/4dd44284c1e64894966c7086f84bfe20 is 175, key is test_row_0/A:col10/1732015017448/Put/seqid=0 2024-11-19T11:16:57,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742381_1557 (size=39749) 2024-11-19T11:16:57,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015077476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:57,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015077477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:57,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015077478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:57,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:57,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:57,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015077588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:57,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:57,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015077588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:57,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:57,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015077588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:57,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:57,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:57,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015077798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:57,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015077798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:57,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015077799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:57,874 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=139, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/4dd44284c1e64894966c7086f84bfe20 2024-11-19T11:16:57,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T11:16:57,878 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:16:57,878 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:16:57,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-19T11:16:57,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T11:16:57,881 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:16:57,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/45afc29dc63e44deb7ce7288caa9bd88 is 50, key is test_row_0/B:col10/1732015017448/Put/seqid=0 2024-11-19T11:16:57,882 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:16:57,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:16:57,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742382_1558 (size=12151) 2024-11-19T11:16:57,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T11:16:58,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T11:16:58,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:58,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:58,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:58,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:58,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:58,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:58,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015078108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:58,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015078108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:58,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:58,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:58,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015078108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:58,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:58,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T11:16:58,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:58,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:58,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:58,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:58,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:58,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:58,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T11:16:58,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/45afc29dc63e44deb7ce7288caa9bd88 2024-11-19T11:16:58,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/bf629a0724fc493b99d5054e63aea8f1 is 50, key is test_row_0/C:col10/1732015017448/Put/seqid=0 2024-11-19T11:16:58,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742383_1559 (size=12151) 2024-11-19T11:16:58,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/bf629a0724fc493b99d5054e63aea8f1 2024-11-19T11:16:58,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/4dd44284c1e64894966c7086f84bfe20 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/4dd44284c1e64894966c7086f84bfe20 2024-11-19T11:16:58,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/4dd44284c1e64894966c7086f84bfe20, entries=200, sequenceid=139, filesize=38.8 K 2024-11-19T11:16:58,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/45afc29dc63e44deb7ce7288caa9bd88 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/45afc29dc63e44deb7ce7288caa9bd88 2024-11-19T11:16:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T11:16:58,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:58,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:58,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:58,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:58,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:16:58,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/45afc29dc63e44deb7ce7288caa9bd88, entries=150, sequenceid=139, filesize=11.9 K 2024-11-19T11:16:58,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/bf629a0724fc493b99d5054e63aea8f1 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/bf629a0724fc493b99d5054e63aea8f1 2024-11-19T11:16:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/bf629a0724fc493b99d5054e63aea8f1, entries=150, sequenceid=139, filesize=11.9 K 2024-11-19T11:16:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 1a9e9054addbd8bfe6392d1e9f0ace31 in 902ms, sequenceid=139, compaction requested=true 2024-11-19T11:16:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:16:58,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a9e9054addbd8bfe6392d1e9f0ace31:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:16:58,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:58,351 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:58,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a9e9054addbd8bfe6392d1e9f0ace31:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:16:58,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:58,351 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:58,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a9e9054addbd8bfe6392d1e9f0ace31:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:16:58,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,352 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:58,352 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 1a9e9054addbd8bfe6392d1e9f0ace31/B is initiating minor compaction (all files) 2024-11-19T11:16:58,352 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a9e9054addbd8bfe6392d1e9f0ace31/B in TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:58,352 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/1d10180dccc748fca1a75a6608389506, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/391a5cec20f746f3bd68b21efd396421, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/45afc29dc63e44deb7ce7288caa9bd88] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp, totalSize=35.5 K 2024-11-19T11:16:58,352 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:58,352 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 1a9e9054addbd8bfe6392d1e9f0ace31/A is initiating minor compaction (all files) 2024-11-19T11:16:58,352 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a9e9054addbd8bfe6392d1e9f0ace31/A in TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:58,352 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/5a6688d029cf49e4a04c98a137ce9c4c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/c01dad77b5384c308d879bbae71880c5, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/4dd44284c1e64894966c7086f84bfe20] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp, totalSize=99.5 K 2024-11-19T11:16:58,353 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:58,353 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. files: [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/5a6688d029cf49e4a04c98a137ce9c4c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/c01dad77b5384c308d879bbae71880c5, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/4dd44284c1e64894966c7086f84bfe20] 2024-11-19T11:16:58,353 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a6688d029cf49e4a04c98a137ce9c4c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732015013768 2024-11-19T11:16:58,353 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d10180dccc748fca1a75a6608389506, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732015013768 2024-11-19T11:16:58,353 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting c01dad77b5384c308d879bbae71880c5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732015014931 2024-11-19T11:16:58,353 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 391a5cec20f746f3bd68b21efd396421, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732015014931 2024-11-19T11:16:58,353 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dd44284c1e64894966c7086f84bfe20, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732015017122 2024-11-19T11:16:58,353 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 45afc29dc63e44deb7ce7288caa9bd88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732015017122 2024-11-19T11:16:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,361 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:16:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,363 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a9e9054addbd8bfe6392d1e9f0ace31#B#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,363 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/e22622a756d3438eabd540a09f291230 is 50, key is test_row_0/B:col10/1732015017448/Put/seqid=0 2024-11-19T11:16:58,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,366 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119557cc120b7374cecba2ca88a625b95b5_1a9e9054addbd8bfe6392d1e9f0ace31 store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:16:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,368 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119557cc120b7374cecba2ca88a625b95b5_1a9e9054addbd8bfe6392d1e9f0ace31, store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:16:58,368 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119557cc120b7374cecba2ca88a625b95b5_1a9e9054addbd8bfe6392d1e9f0ace31 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:16:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742384_1560 (size=12459) 2024-11-19T11:16:58,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742385_1561 (size=4469) 2024-11-19T11:16:58,406 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a9e9054addbd8bfe6392d1e9f0ace31#A#compaction#476 average throughput is 0.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,407 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/ab8a81300c7048cb8096df003e7d293b is 175, key is test_row_0/A:col10/1732015017448/Put/seqid=0 2024-11-19T11:16:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742386_1562 (size=31413) 2024-11-19T11:16:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,434 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/ab8a81300c7048cb8096df003e7d293b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/ab8a81300c7048cb8096df003e7d293b 2024-11-19T11:16:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,440 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a9e9054addbd8bfe6392d1e9f0ace31/A of 1a9e9054addbd8bfe6392d1e9f0ace31 into ab8a81300c7048cb8096df003e7d293b(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:58,440 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:16:58,440 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., storeName=1a9e9054addbd8bfe6392d1e9f0ace31/A, priority=13, startTime=1732015018351; duration=0sec 2024-11-19T11:16:58,440 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:16:58,440 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a9e9054addbd8bfe6392d1e9f0ace31:A 2024-11-19T11:16:58,440 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:16:58,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,441 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:16:58,441 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 1a9e9054addbd8bfe6392d1e9f0ace31/C is initiating minor compaction (all files) 2024-11-19T11:16:58,441 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a9e9054addbd8bfe6392d1e9f0ace31/C in TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:58,441 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/b165fd7779e94f128883cdb7a84e806a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/03c8e20a43cd4ad9ab7364a9b828abe5, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/bf629a0724fc493b99d5054e63aea8f1] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp, totalSize=35.5 K 2024-11-19T11:16:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,441 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting b165fd7779e94f128883cdb7a84e806a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732015013768 2024-11-19T11:16:58,442 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03c8e20a43cd4ad9ab7364a9b828abe5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732015014931 2024-11-19T11:16:58,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,442 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf629a0724fc493b99d5054e63aea8f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732015017122 2024-11-19T11:16:58,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,458 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a9e9054addbd8bfe6392d1e9f0ace31#C#compaction#478 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:16:58,458 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/400b9a58c4c5479a927c03a04274fb49 is 50, key is test_row_0/C:col10/1732015017448/Put/seqid=0 2024-11-19T11:16:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742387_1563 (size=12459) 2024-11-19T11:16:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,467 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/400b9a58c4c5479a927c03a04274fb49 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/400b9a58c4c5479a927c03a04274fb49 2024-11-19T11:16:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,471 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a9e9054addbd8bfe6392d1e9f0ace31/C of 1a9e9054addbd8bfe6392d1e9f0ace31 into 400b9a58c4c5479a927c03a04274fb49(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:58,471 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:16:58,471 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., storeName=1a9e9054addbd8bfe6392d1e9f0ace31/C, priority=13, startTime=1732015018351; duration=0sec 2024-11-19T11:16:58,471 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:58,471 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a9e9054addbd8bfe6392d1e9f0ace31:C 2024-11-19T11:16:58,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T11:16:58,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:58,495 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 1a9e9054addbd8bfe6392d1e9f0ace31 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T11:16:58,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=A 2024-11-19T11:16:58,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:58,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=B 2024-11-19T11:16:58,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:58,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=C 2024-11-19T11:16:58,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:58,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f16f9e7b413c496085d5c42014941fd0_1a9e9054addbd8bfe6392d1e9f0ace31 is 50, key is test_row_0/A:col10/1732015017477/Put/seqid=0 2024-11-19T11:16:58,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742388_1564 (size=9814) 2024-11-19T11:16:58,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T11:16:58,510 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f16f9e7b413c496085d5c42014941fd0_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f16f9e7b413c496085d5c42014941fd0_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:58,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/27ef460163494342b30559c7ee0330b2, store: [table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:16:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/27ef460163494342b30559c7ee0330b2 is 175, key is test_row_0/A:col10/1732015017477/Put/seqid=0 2024-11-19T11:16:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742389_1565 (size=22461) 2024-11-19T11:16:58,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:58,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:16:58,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:58,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:58,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015078645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:58,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015078649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:58,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015078650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:58,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015078758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:58,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015078758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:58,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015078767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:58,802 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/e22622a756d3438eabd540a09f291230 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/e22622a756d3438eabd540a09f291230 2024-11-19T11:16:58,808 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a9e9054addbd8bfe6392d1e9f0ace31/B of 1a9e9054addbd8bfe6392d1e9f0ace31 into e22622a756d3438eabd540a09f291230(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:16:58,808 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:16:58,808 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., storeName=1a9e9054addbd8bfe6392d1e9f0ace31/B, priority=13, startTime=1732015018351; duration=0sec 2024-11-19T11:16:58,808 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:16:58,808 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a9e9054addbd8bfe6392d1e9f0ace31:B 2024-11-19T11:16:58,917 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/27ef460163494342b30559c7ee0330b2 2024-11-19T11:16:58,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/6b51565e4c104a72a09085d99bb0d298 is 50, key is test_row_0/B:col10/1732015017477/Put/seqid=0 2024-11-19T11:16:58,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742390_1566 (size=9757) 2024-11-19T11:16:58,963 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/6b51565e4c104a72a09085d99bb0d298 2024-11-19T11:16:58,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:58,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015078967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:58,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:58,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:58,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015078972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:58,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:58,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/2815a2e7e0ad4281b4fc9fa888f90023 is 50, key is test_row_0/C:col10/1732015017477/Put/seqid=0 2024-11-19T11:16:58,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015078978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:59,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T11:16:59,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742391_1567 (size=9757) 2024-11-19T11:16:59,048 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/2815a2e7e0ad4281b4fc9fa888f90023 2024-11-19T11:16:59,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/27ef460163494342b30559c7ee0330b2 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/27ef460163494342b30559c7ee0330b2 2024-11-19T11:16:59,060 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/27ef460163494342b30559c7ee0330b2, entries=100, sequenceid=156, filesize=21.9 K 2024-11-19T11:16:59,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/6b51565e4c104a72a09085d99bb0d298 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/6b51565e4c104a72a09085d99bb0d298 2024-11-19T11:16:59,068 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/6b51565e4c104a72a09085d99bb0d298, entries=100, sequenceid=156, filesize=9.5 K 2024-11-19T11:16:59,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/2815a2e7e0ad4281b4fc9fa888f90023 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/2815a2e7e0ad4281b4fc9fa888f90023 2024-11-19T11:16:59,075 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/2815a2e7e0ad4281b4fc9fa888f90023, entries=100, sequenceid=156, filesize=9.5 K 2024-11-19T11:16:59,076 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1a9e9054addbd8bfe6392d1e9f0ace31 in 581ms, sequenceid=156, compaction requested=false 2024-11-19T11:16:59,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:16:59,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:16:59,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-19T11:16:59,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-19T11:16:59,079 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-19T11:16:59,079 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1950 sec 2024-11-19T11:16:59,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.2010 sec 2024-11-19T11:16:59,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:59,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a9e9054addbd8bfe6392d1e9f0ace31 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T11:16:59,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=A 2024-11-19T11:16:59,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:59,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=B 2024-11-19T11:16:59,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:59,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=C 2024-11-19T11:16:59,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:16:59,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:59,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015079295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015079296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:59,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:59,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111919123f2bc8c04c178afbe83c560fa4d7_1a9e9054addbd8bfe6392d1e9f0ace31 is 50, key is test_row_0/A:col10/1732015018648/Put/seqid=0 2024-11-19T11:16:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015079297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:59,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:59,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742392_1568 (size=12304) 2024-11-19T11:16:59,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:59,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015079407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:59,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:59,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015079407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:59,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:59,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:59,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:59,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015079407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:59,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:59,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015079618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:59,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:59,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015079618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:59,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:59,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:59,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:59,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015079618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:59,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:59,734 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:16:59,737 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111919123f2bc8c04c178afbe83c560fa4d7_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111919123f2bc8c04c178afbe83c560fa4d7_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:16:59,737 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/1b02a8cef68f47f5ad9b87229476719b, store: [table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:16:59,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/1b02a8cef68f47f5ad9b87229476719b is 175, key is test_row_0/A:col10/1732015018648/Put/seqid=0 2024-11-19T11:16:59,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742393_1569 (size=31105) 2024-11-19T11:16:59,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55098 deadline: 1732015079838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:59,840 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_2' in 1a9e9054addbd8bfe6392d1e9f0ace31 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:59,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55080 deadline: 1732015079857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:59,862 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 16c88966c0af,35833,1732014855640 for 'test_row_0' in 1a9e9054addbd8bfe6392d1e9f0ace31 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T11:16:59,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:59,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015079928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:59,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:59,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015079928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:59,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:16:59,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:16:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015079928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:16:59,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:16:59,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T11:17:00,037 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:17:00,037 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:17:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-19T11:17:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T11:17:00,040 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:17:00,040 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:17:00,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:17:00,141 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/1b02a8cef68f47f5ad9b87229476719b 2024-11-19T11:17:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T11:17:00,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/04391d7d576240bba230eb582976015b is 50, key is test_row_0/B:col10/1732015018648/Put/seqid=0 2024-11-19T11:17:00,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742394_1570 (size=12151) 2024-11-19T11:17:00,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T11:17:00,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:00,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:17:00,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:00,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:00,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:00,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T11:17:00,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:00,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:17:00,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:00,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:00,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T11:17:00,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015080438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:00,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015080438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:00,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015080438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:00,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T11:17:00,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:00,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:17:00,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:00,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:00,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:00,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:00,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/04391d7d576240bba230eb582976015b 2024-11-19T11:17:00,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/9b65d114e78c45168022d433a45e5782 is 50, key is test_row_0/C:col10/1732015018648/Put/seqid=0 2024-11-19T11:17:00,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742395_1571 (size=12151) 2024-11-19T11:17:00,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/9b65d114e78c45168022d433a45e5782 2024-11-19T11:17:00,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/1b02a8cef68f47f5ad9b87229476719b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/1b02a8cef68f47f5ad9b87229476719b 2024-11-19T11:17:00,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/1b02a8cef68f47f5ad9b87229476719b, entries=150, sequenceid=175, filesize=30.4 K 2024-11-19T11:17:00,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/04391d7d576240bba230eb582976015b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/04391d7d576240bba230eb582976015b 2024-11-19T11:17:00,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/04391d7d576240bba230eb582976015b, entries=150, sequenceid=175, filesize=11.9 K 2024-11-19T11:17:00,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/9b65d114e78c45168022d433a45e5782 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/9b65d114e78c45168022d433a45e5782 2024-11-19T11:17:00,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/9b65d114e78c45168022d433a45e5782, entries=150, sequenceid=175, filesize=11.9 K 2024-11-19T11:17:00,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1a9e9054addbd8bfe6392d1e9f0ace31 in 1363ms, sequenceid=175, compaction requested=true 2024-11-19T11:17:00,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:17:00,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a9e9054addbd8bfe6392d1e9f0ace31:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:17:00,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:17:00,643 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:17:00,643 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:17:00,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a9e9054addbd8bfe6392d1e9f0ace31:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:17:00,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:17:00,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a9e9054addbd8bfe6392d1e9f0ace31:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:17:00,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:17:00,644 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 84979 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:17:00,644 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 1a9e9054addbd8bfe6392d1e9f0ace31/A is initiating minor compaction (all files) 2024-11-19T11:17:00,644 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a9e9054addbd8bfe6392d1e9f0ace31/A in TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:00,644 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/ab8a81300c7048cb8096df003e7d293b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/27ef460163494342b30559c7ee0330b2, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/1b02a8cef68f47f5ad9b87229476719b] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp, totalSize=83.0 K 2024-11-19T11:17:00,644 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:00,644 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. files: [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/ab8a81300c7048cb8096df003e7d293b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/27ef460163494342b30559c7ee0330b2, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/1b02a8cef68f47f5ad9b87229476719b] 2024-11-19T11:17:00,647 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:17:00,647 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 1a9e9054addbd8bfe6392d1e9f0ace31/B is initiating minor compaction (all files) 2024-11-19T11:17:00,647 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a9e9054addbd8bfe6392d1e9f0ace31/B in TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:00,647 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/e22622a756d3438eabd540a09f291230, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/6b51565e4c104a72a09085d99bb0d298, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/04391d7d576240bba230eb582976015b] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp, totalSize=33.6 K 2024-11-19T11:17:00,648 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab8a81300c7048cb8096df003e7d293b, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732015017122 2024-11-19T11:17:00,648 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting e22622a756d3438eabd540a09f291230, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732015017122 2024-11-19T11:17:00,649 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b51565e4c104a72a09085d99bb0d298, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732015017470 2024-11-19T11:17:00,649 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27ef460163494342b30559c7ee0330b2, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732015017470 2024-11-19T11:17:00,650 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 04391d7d576240bba230eb582976015b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732015018646 2024-11-19T11:17:00,650 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b02a8cef68f47f5ad9b87229476719b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732015018646 2024-11-19T11:17:00,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T11:17:00,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:00,653 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 1a9e9054addbd8bfe6392d1e9f0ace31 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T11:17:00,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=A 2024-11-19T11:17:00,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:17:00,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=B 2024-11-19T11:17:00,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:17:00,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=C 2024-11-19T11:17:00,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:17:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T11:17:00,682 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a9e9054addbd8bfe6392d1e9f0ace31#B#compaction#485 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:17:00,683 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/7627225f1a5443f1b5cb75db08391a23 is 50, key is test_row_0/B:col10/1732015018648/Put/seqid=0 2024-11-19T11:17:00,689 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:17:00,693 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111993afc41fdbec4e83bea2c64c071c972a_1a9e9054addbd8bfe6392d1e9f0ace31 store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:17:00,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ed3353ff193e463ebb51ee7d55e52e45_1a9e9054addbd8bfe6392d1e9f0ace31 is 50, key is test_row_0/A:col10/1732015019293/Put/seqid=0 2024-11-19T11:17:00,694 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111993afc41fdbec4e83bea2c64c071c972a_1a9e9054addbd8bfe6392d1e9f0ace31, store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:17:00,695 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111993afc41fdbec4e83bea2c64c071c972a_1a9e9054addbd8bfe6392d1e9f0ace31 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:17:00,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742396_1572 (size=12561) 2024-11-19T11:17:00,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742397_1573 (size=12304) 2024-11-19T11:17:00,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742398_1574 (size=4469) 2024-11-19T11:17:01,112 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/7627225f1a5443f1b5cb75db08391a23 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/7627225f1a5443f1b5cb75db08391a23 2024-11-19T11:17:01,116 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a9e9054addbd8bfe6392d1e9f0ace31/B of 1a9e9054addbd8bfe6392d1e9f0ace31 into 7627225f1a5443f1b5cb75db08391a23(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:17:01,116 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:17:01,116 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., storeName=1a9e9054addbd8bfe6392d1e9f0ace31/B, priority=13, startTime=1732015020643; duration=0sec 2024-11-19T11:17:01,116 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:17:01,116 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a9e9054addbd8bfe6392d1e9f0ace31:B 2024-11-19T11:17:01,116 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:17:01,117 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:17:01,117 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 1a9e9054addbd8bfe6392d1e9f0ace31/C is initiating minor compaction (all files) 2024-11-19T11:17:01,117 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a9e9054addbd8bfe6392d1e9f0ace31/C in TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:01,117 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/400b9a58c4c5479a927c03a04274fb49, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/2815a2e7e0ad4281b4fc9fa888f90023, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/9b65d114e78c45168022d433a45e5782] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp, totalSize=33.6 K 2024-11-19T11:17:01,118 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 400b9a58c4c5479a927c03a04274fb49, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732015017122 2024-11-19T11:17:01,118 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 2815a2e7e0ad4281b4fc9fa888f90023, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732015017470 2024-11-19T11:17:01,118 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b65d114e78c45168022d433a45e5782, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732015018646 2024-11-19T11:17:01,123 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a9e9054addbd8bfe6392d1e9f0ace31#C#compaction#488 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:17:01,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:17:01,123 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/64f83db738fb4ffd91ba1bb0b97e81ef is 50, key is test_row_0/C:col10/1732015018648/Put/seqid=0 2024-11-19T11:17:01,126 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ed3353ff193e463ebb51ee7d55e52e45_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ed3353ff193e463ebb51ee7d55e52e45_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:01,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742399_1575 (size=12561) 2024-11-19T11:17:01,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/640bdb92966f469ea3907bf53d6e548b, store: [table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:17:01,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/640bdb92966f469ea3907bf53d6e548b is 175, key is test_row_0/A:col10/1732015019293/Put/seqid=0 2024-11-19T11:17:01,130 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/64f83db738fb4ffd91ba1bb0b97e81ef as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/64f83db738fb4ffd91ba1bb0b97e81ef 2024-11-19T11:17:01,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742400_1576 (size=31105) 2024-11-19T11:17:01,131 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a9e9054addbd8bfe6392d1e9f0ace31#A#compaction#486 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:17:01,131 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/ccebe494bc084c31898c12df4f65ff93 is 175, key is test_row_0/A:col10/1732015018648/Put/seqid=0 2024-11-19T11:17:01,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742401_1577 (size=31515) 2024-11-19T11:17:01,136 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a9e9054addbd8bfe6392d1e9f0ace31/C of 1a9e9054addbd8bfe6392d1e9f0ace31 into 64f83db738fb4ffd91ba1bb0b97e81ef(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:17:01,136 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:17:01,136 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., storeName=1a9e9054addbd8bfe6392d1e9f0ace31/C, priority=13, startTime=1732015020643; duration=0sec 2024-11-19T11:17:01,136 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:17:01,136 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a9e9054addbd8bfe6392d1e9f0ace31:C 2024-11-19T11:17:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T11:17:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:01,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:17:01,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015081481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:01,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:01,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:01,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015081483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:01,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015081485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:01,531 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/640bdb92966f469ea3907bf53d6e548b 2024-11-19T11:17:01,540 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/ccebe494bc084c31898c12df4f65ff93 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/ccebe494bc084c31898c12df4f65ff93 2024-11-19T11:17:01,545 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a9e9054addbd8bfe6392d1e9f0ace31/A of 1a9e9054addbd8bfe6392d1e9f0ace31 into ccebe494bc084c31898c12df4f65ff93(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:17:01,545 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:17:01,545 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., storeName=1a9e9054addbd8bfe6392d1e9f0ace31/A, priority=13, startTime=1732015020642; duration=0sec 2024-11-19T11:17:01,545 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:17:01,545 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a9e9054addbd8bfe6392d1e9f0ace31:A 2024-11-19T11:17:01,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/6a73a649b8be4c2f882c78b8080bf492 is 50, key is test_row_0/B:col10/1732015019293/Put/seqid=0 2024-11-19T11:17:01,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015081598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:01,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015081598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:01,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015081599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:01,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:01,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:01,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:01,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742402_1578 (size=12151) 2024-11-19T11:17:01,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015081808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:01,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015081808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:01,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015081808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:01,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:01,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:01,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:02,018 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/6a73a649b8be4c2f882c78b8080bf492 2024-11-19T11:17:02,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/d6a3908c3bb448fbacc2e91003e3b87f is 50, key is test_row_0/C:col10/1732015019293/Put/seqid=0 2024-11-19T11:17:02,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742403_1579 (size=12151) 2024-11-19T11:17:02,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015082117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:02,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015082117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:02,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015082117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:02,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T11:17:02,427 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/d6a3908c3bb448fbacc2e91003e3b87f 2024-11-19T11:17:02,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/640bdb92966f469ea3907bf53d6e548b as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/640bdb92966f469ea3907bf53d6e548b 2024-11-19T11:17:02,434 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/640bdb92966f469ea3907bf53d6e548b, entries=150, sequenceid=189, filesize=30.4 K 2024-11-19T11:17:02,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/6a73a649b8be4c2f882c78b8080bf492 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/6a73a649b8be4c2f882c78b8080bf492 2024-11-19T11:17:02,437 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/6a73a649b8be4c2f882c78b8080bf492, entries=150, sequenceid=189, filesize=11.9 K 2024-11-19T11:17:02,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/d6a3908c3bb448fbacc2e91003e3b87f as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/d6a3908c3bb448fbacc2e91003e3b87f 2024-11-19T11:17:02,441 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/d6a3908c3bb448fbacc2e91003e3b87f, entries=150, sequenceid=189, filesize=11.9 K 2024-11-19T11:17:02,442 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1a9e9054addbd8bfe6392d1e9f0ace31 in 1789ms, sequenceid=189, compaction requested=false 2024-11-19T11:17:02,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:17:02,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:02,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-19T11:17:02,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-19T11:17:02,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-19T11:17:02,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4030 sec 2024-11-19T11:17:02,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.4070 sec 2024-11-19T11:17:02,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:02,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a9e9054addbd8bfe6392d1e9f0ace31 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T11:17:02,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=A 2024-11-19T11:17:02,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:17:02,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=B 2024-11-19T11:17:02,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:17:02,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=C 2024-11-19T11:17:02,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:17:02,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111933abf6b121ce405f93e74291dc69e675_1a9e9054addbd8bfe6392d1e9f0ace31 is 50, key is test_row_0/A:col10/1732015022627/Put/seqid=0 2024-11-19T11:17:02,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742404_1580 (size=12304) 2024-11-19T11:17:02,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015082650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:02,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:02,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:02,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015082651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:02,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:02,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:02,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015082652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:02,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:02,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:02,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015082757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:02,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:02,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015082768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:02,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:02,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015082768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:02,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:02,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:02,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015082967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:02,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:02,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015082978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:02,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:02,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015082978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:02,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:02,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:02,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:03,046 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:17:03,048 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111933abf6b121ce405f93e74291dc69e675_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111933abf6b121ce405f93e74291dc69e675_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:03,049 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/5390836a18b64fd885b4d23e10a8b1a4, store: [table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:17:03,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/5390836a18b64fd885b4d23e10a8b1a4 is 175, key is test_row_0/A:col10/1732015022627/Put/seqid=0 2024-11-19T11:17:03,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742405_1581 (size=31105) 2024-11-19T11:17:03,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015083277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:03,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:03,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:03,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:03,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015083287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:03,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:03,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:03,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015083288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:03,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:03,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:03,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:03,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:03,458 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/5390836a18b64fd885b4d23e10a8b1a4 2024-11-19T11:17:03,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/9f0a650cff79487bb89346457ce1411a is 50, key is test_row_0/B:col10/1732015022627/Put/seqid=0 2024-11-19T11:17:03,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742406_1582 (size=12151) 2024-11-19T11:17:03,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:03,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015083787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:03,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:03,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:03,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:03,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015083797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:03,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:03,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:03,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:03,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015083797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:03,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/9f0a650cff79487bb89346457ce1411a 2024-11-19T11:17:03,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/345532e00c99414f92af937897d2d203 is 50, key is test_row_0/C:col10/1732015022627/Put/seqid=0 2024-11-19T11:17:03,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742407_1583 (size=12151) 2024-11-19T11:17:04,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T11:17:04,208 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:17:04,208 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:17:04,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-19T11:17:04,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T11:17:04,210 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:17:04,210 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:17:04,210 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:17:04,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/345532e00c99414f92af937897d2d203 2024-11-19T11:17:04,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/5390836a18b64fd885b4d23e10a8b1a4 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/5390836a18b64fd885b4d23e10a8b1a4 2024-11-19T11:17:04,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/5390836a18b64fd885b4d23e10a8b1a4, entries=150, sequenceid=211, filesize=30.4 K 2024-11-19T11:17:04,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/9f0a650cff79487bb89346457ce1411a as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/9f0a650cff79487bb89346457ce1411a 2024-11-19T11:17:04,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/9f0a650cff79487bb89346457ce1411a, entries=150, sequenceid=211, filesize=11.9 K 2024-11-19T11:17:04,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/345532e00c99414f92af937897d2d203 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/345532e00c99414f92af937897d2d203 2024-11-19T11:17:04,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/345532e00c99414f92af937897d2d203, entries=150, sequenceid=211, filesize=11.9 K 2024-11-19T11:17:04,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 1a9e9054addbd8bfe6392d1e9f0ace31 in 1679ms, sequenceid=211, compaction requested=true 2024-11-19T11:17:04,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:17:04,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a9e9054addbd8bfe6392d1e9f0ace31:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:17:04,308 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:17:04,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:17:04,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a9e9054addbd8bfe6392d1e9f0ace31:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:17:04,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:17:04,308 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:17:04,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a9e9054addbd8bfe6392d1e9f0ace31:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:17:04,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:17:04,309 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:17:04,309 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:17:04,309 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 1a9e9054addbd8bfe6392d1e9f0ace31/B is initiating minor compaction (all files) 2024-11-19T11:17:04,309 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 1a9e9054addbd8bfe6392d1e9f0ace31/A is initiating minor compaction (all files) 2024-11-19T11:17:04,309 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a9e9054addbd8bfe6392d1e9f0ace31/B in TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:04,309 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a9e9054addbd8bfe6392d1e9f0ace31/A in TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:04,309 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/7627225f1a5443f1b5cb75db08391a23, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/6a73a649b8be4c2f882c78b8080bf492, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/9f0a650cff79487bb89346457ce1411a] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp, totalSize=36.0 K 2024-11-19T11:17:04,309 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/ccebe494bc084c31898c12df4f65ff93, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/640bdb92966f469ea3907bf53d6e548b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/5390836a18b64fd885b4d23e10a8b1a4] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp, totalSize=91.5 K 2024-11-19T11:17:04,309 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:04,309 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. files: [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/ccebe494bc084c31898c12df4f65ff93, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/640bdb92966f469ea3907bf53d6e548b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/5390836a18b64fd885b4d23e10a8b1a4] 2024-11-19T11:17:04,309 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 7627225f1a5443f1b5cb75db08391a23, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732015018646 2024-11-19T11:17:04,309 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccebe494bc084c31898c12df4f65ff93, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732015018646 2024-11-19T11:17:04,310 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a73a649b8be4c2f882c78b8080bf492, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732015019291 2024-11-19T11:17:04,310 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 640bdb92966f469ea3907bf53d6e548b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732015019291 2024-11-19T11:17:04,310 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f0a650cff79487bb89346457ce1411a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732015021478 2024-11-19T11:17:04,310 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5390836a18b64fd885b4d23e10a8b1a4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732015021478 2024-11-19T11:17:04,314 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:17:04,315 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119f4a220202e7e45bcaa1bec5e333d784d_1a9e9054addbd8bfe6392d1e9f0ace31 store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:17:04,316 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a9e9054addbd8bfe6392d1e9f0ace31#B#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:17:04,317 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119f4a220202e7e45bcaa1bec5e333d784d_1a9e9054addbd8bfe6392d1e9f0ace31, store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:17:04,317 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f4a220202e7e45bcaa1bec5e333d784d_1a9e9054addbd8bfe6392d1e9f0ace31 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:17:04,317 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/2f7aab2807474c1d8189592cf19ee350 is 50, key is test_row_0/B:col10/1732015022627/Put/seqid=0 2024-11-19T11:17:04,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T11:17:04,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742408_1584 (size=4469) 2024-11-19T11:17:04,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742409_1585 (size=12663) 2024-11-19T11:17:04,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T11:17:04,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:04,362 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 1a9e9054addbd8bfe6392d1e9f0ace31 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T11:17:04,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=A 2024-11-19T11:17:04,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:17:04,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=B 2024-11-19T11:17:04,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:17:04,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=C 2024-11-19T11:17:04,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:17:04,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ef411e763c1c47b995a69d74b8b06010_1a9e9054addbd8bfe6392d1e9f0ace31 is 50, key is test_row_0/A:col10/1732015022643/Put/seqid=0 2024-11-19T11:17:04,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742410_1586 (size=12304) 2024-11-19T11:17:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T11:17:04,722 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a9e9054addbd8bfe6392d1e9f0ace31#A#compaction#494 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:17:04,722 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/7f788c0c40e64f3f85bbc8b1436e38ec is 175, key is test_row_0/A:col10/1732015022627/Put/seqid=0 2024-11-19T11:17:04,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742411_1587 (size=31617) 2024-11-19T11:17:04,728 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/2f7aab2807474c1d8189592cf19ee350 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/2f7aab2807474c1d8189592cf19ee350 2024-11-19T11:17:04,732 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a9e9054addbd8bfe6392d1e9f0ace31/B of 1a9e9054addbd8bfe6392d1e9f0ace31 into 2f7aab2807474c1d8189592cf19ee350(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:17:04,732 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:17:04,732 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., storeName=1a9e9054addbd8bfe6392d1e9f0ace31/B, priority=13, startTime=1732015024308; duration=0sec 2024-11-19T11:17:04,732 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:17:04,732 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a9e9054addbd8bfe6392d1e9f0ace31:B 2024-11-19T11:17:04,732 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:17:04,733 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:17:04,733 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 1a9e9054addbd8bfe6392d1e9f0ace31/C is initiating minor compaction (all files) 2024-11-19T11:17:04,733 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a9e9054addbd8bfe6392d1e9f0ace31/C in TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:04,733 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/64f83db738fb4ffd91ba1bb0b97e81ef, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/d6a3908c3bb448fbacc2e91003e3b87f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/345532e00c99414f92af937897d2d203] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp, totalSize=36.0 K 2024-11-19T11:17:04,734 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 64f83db738fb4ffd91ba1bb0b97e81ef, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732015018646 2024-11-19T11:17:04,734 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting d6a3908c3bb448fbacc2e91003e3b87f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732015019291 2024-11-19T11:17:04,734 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 345532e00c99414f92af937897d2d203, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732015021478 2024-11-19T11:17:04,740 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a9e9054addbd8bfe6392d1e9f0ace31#C#compaction#497 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:17:04,740 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/37a543488cb64adca91cd0832db6b748 is 50, key is test_row_0/C:col10/1732015022627/Put/seqid=0 2024-11-19T11:17:04,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742412_1588 (size=12663) 2024-11-19T11:17:04,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:17:04,774 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ef411e763c1c47b995a69d74b8b06010_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ef411e763c1c47b995a69d74b8b06010_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:04,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/f84a0cc330f441c88d140b2fac283d64, store: [table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:17:04,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/f84a0cc330f441c88d140b2fac283d64 is 175, key is test_row_0/A:col10/1732015022643/Put/seqid=0 2024-11-19T11:17:04,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742413_1589 (size=31105) 2024-11-19T11:17:04,779 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/f84a0cc330f441c88d140b2fac283d64 2024-11-19T11:17:04,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/eb8c1e591f3742dabec8abcc3b6845ea is 50, key is test_row_0/B:col10/1732015022643/Put/seqid=0 2024-11-19T11:17:04,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742414_1590 (size=12151) 2024-11-19T11:17:04,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:04,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:17:04,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:04,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015084830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:04,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:04,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:04,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:04,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:04,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015084832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:04,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:04,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:04,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:04,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T11:17:04,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:04,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015084835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:04,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:04,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015084947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:04,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:04,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:04,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015084947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:04,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:04,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:04,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:04,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:04,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:04,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:04,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015084947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:04,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:04,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:04,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:05,129 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/7f788c0c40e64f3f85bbc8b1436e38ec as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/7f788c0c40e64f3f85bbc8b1436e38ec 2024-11-19T11:17:05,133 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a9e9054addbd8bfe6392d1e9f0ace31/A of 1a9e9054addbd8bfe6392d1e9f0ace31 into 7f788c0c40e64f3f85bbc8b1436e38ec(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:17:05,133 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:17:05,133 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., storeName=1a9e9054addbd8bfe6392d1e9f0ace31/A, priority=13, startTime=1732015024308; duration=0sec 2024-11-19T11:17:05,133 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:17:05,133 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a9e9054addbd8bfe6392d1e9f0ace31:A 2024-11-19T11:17:05,147 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/37a543488cb64adca91cd0832db6b748 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/37a543488cb64adca91cd0832db6b748 2024-11-19T11:17:05,151 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a9e9054addbd8bfe6392d1e9f0ace31/C of 1a9e9054addbd8bfe6392d1e9f0ace31 into 37a543488cb64adca91cd0832db6b748(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:17:05,151 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:17:05,151 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., storeName=1a9e9054addbd8bfe6392d1e9f0ace31/C, priority=13, startTime=1732015024308; duration=0sec 2024-11-19T11:17:05,151 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:17:05,151 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a9e9054addbd8bfe6392d1e9f0ace31:C 2024-11-19T11:17:05,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015085157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:05,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015085157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:05,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015085157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:05,186 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/eb8c1e591f3742dabec8abcc3b6845ea 2024-11-19T11:17:05,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/fd47b0f8a03f4c67ba2e780a44956ce1 is 50, key is test_row_0/C:col10/1732015022643/Put/seqid=0 2024-11-19T11:17:05,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742415_1591 (size=12151) 2024-11-19T11:17:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T11:17:05,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015085468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:05,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:05,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:05,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015085468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:05,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:05,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:05,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:05,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:05,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:05,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015085468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:05,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:05,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:05,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:05,600 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/fd47b0f8a03f4c67ba2e780a44956ce1 2024-11-19T11:17:05,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/f84a0cc330f441c88d140b2fac283d64 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/f84a0cc330f441c88d140b2fac283d64 2024-11-19T11:17:05,606 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/f84a0cc330f441c88d140b2fac283d64, entries=150, sequenceid=225, filesize=30.4 K 2024-11-19T11:17:05,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/eb8c1e591f3742dabec8abcc3b6845ea as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/eb8c1e591f3742dabec8abcc3b6845ea 2024-11-19T11:17:05,609 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/eb8c1e591f3742dabec8abcc3b6845ea, entries=150, sequenceid=225, filesize=11.9 K 2024-11-19T11:17:05,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/fd47b0f8a03f4c67ba2e780a44956ce1 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/fd47b0f8a03f4c67ba2e780a44956ce1 2024-11-19T11:17:05,612 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/fd47b0f8a03f4c67ba2e780a44956ce1, entries=150, sequenceid=225, filesize=11.9 K 2024-11-19T11:17:05,613 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1a9e9054addbd8bfe6392d1e9f0ace31 in 1250ms, sequenceid=225, compaction requested=false 2024-11-19T11:17:05,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:17:05,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:05,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-19T11:17:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-19T11:17:05,615 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-19T11:17:05,615 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4040 sec 2024-11-19T11:17:05,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.4080 sec 2024-11-19T11:17:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:05,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a9e9054addbd8bfe6392d1e9f0ace31 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T11:17:05,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=A 2024-11-19T11:17:05,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:17:05,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=B 2024-11-19T11:17:05,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:17:05,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=C 2024-11-19T11:17:05,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:17:05,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a0d90f9a9d21437ba643751bfa4fbdf5_1a9e9054addbd8bfe6392d1e9f0ace31 is 50, key is test_row_0/A:col10/1732015025980/Put/seqid=0 2024-11-19T11:17:06,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742416_1592 (size=12304) 2024-11-19T11:17:06,022 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:17:06,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015086017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:06,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015086018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:06,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015086020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:06,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:06,027 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a0d90f9a9d21437ba643751bfa4fbdf5_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a0d90f9a9d21437ba643751bfa4fbdf5_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:06,029 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/fbfa56ad6e6c459bafe4ec1de9fb5ab7, store: [table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:17:06,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/fbfa56ad6e6c459bafe4ec1de9fb5ab7 is 175, key is test_row_0/A:col10/1732015025980/Put/seqid=0 2024-11-19T11:17:06,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742417_1593 (size=31105) 2024-11-19T11:17:06,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015086127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:06,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:06,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015086137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:06,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:06,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015086137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:06,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015086337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:06,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015086348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:06,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:06,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015086348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:06,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:06,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T11:17:06,357 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:17:06,357 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T11:17:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-19T11:17:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T11:17:06,359 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T11:17:06,359 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T11:17:06,359 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T11:17:06,435 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/fbfa56ad6e6c459bafe4ec1de9fb5ab7 2024-11-19T11:17:06,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/487f5949c8644a5abeedf2cf8f032e92 is 50, key is test_row_0/B:col10/1732015025980/Put/seqid=0 2024-11-19T11:17:06,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742418_1594 (size=12151) 2024-11-19T11:17:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T11:17:06,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T11:17:06,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:06,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:17:06,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:06,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:06,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:06,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015086647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:06,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:06,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015086657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:06,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:06,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:06,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015086657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:06,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:06,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:06,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T11:17:06,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:06,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:17:06,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:06,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:06,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:06,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T11:17:06,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T11:17:06,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:06,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:17:06,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:06,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:06,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:06,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:06,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/487f5949c8644a5abeedf2cf8f032e92 2024-11-19T11:17:06,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/8c7086403a3546d4a8cc9c121110776a is 50, key is test_row_0/C:col10/1732015025980/Put/seqid=0 2024-11-19T11:17:06,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742419_1595 (size=12151) 2024-11-19T11:17:06,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T11:17:06,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:06,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:17:06,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:06,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:06,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T11:17:07,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T11:17:07,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:07,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:17:07,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:07,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:07,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:07,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55084 deadline: 1732015087157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:07,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:07,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:07,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:07,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55088 deadline: 1732015087168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:07,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:17:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55136 deadline: 1732015087168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 2024-11-19T11:17:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:07,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 , the old value is region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:07,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:17:07,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., hostname=16c88966c0af,35833,1732014855640, seqNum=5 because the exception is null or not the one we care about 2024-11-19T11:17:07,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/8c7086403a3546d4a8cc9c121110776a 2024-11-19T11:17:07,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/fbfa56ad6e6c459bafe4ec1de9fb5ab7 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/fbfa56ad6e6c459bafe4ec1de9fb5ab7 2024-11-19T11:17:07,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/fbfa56ad6e6c459bafe4ec1de9fb5ab7, entries=150, sequenceid=248, filesize=30.4 K 2024-11-19T11:17:07,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/487f5949c8644a5abeedf2cf8f032e92 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/487f5949c8644a5abeedf2cf8f032e92 2024-11-19T11:17:07,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/487f5949c8644a5abeedf2cf8f032e92, entries=150, sequenceid=248, filesize=11.9 K 2024-11-19T11:17:07,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/8c7086403a3546d4a8cc9c121110776a as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/8c7086403a3546d4a8cc9c121110776a 2024-11-19T11:17:07,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T11:17:07,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:07,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:17:07,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:07,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:07,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:07,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/8c7086403a3546d4a8cc9c121110776a, entries=150, sequenceid=248, filesize=11.9 K 2024-11-19T11:17:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:17:07,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1a9e9054addbd8bfe6392d1e9f0ace31 in 1295ms, sequenceid=248, compaction requested=true 2024-11-19T11:17:07,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:17:07,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a9e9054addbd8bfe6392d1e9f0ace31:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:17:07,275 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:17:07,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:17:07,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a9e9054addbd8bfe6392d1e9f0ace31:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:17:07,276 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:17:07,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:17:07,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a9e9054addbd8bfe6392d1e9f0ace31:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T11:17:07,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:17:07,276 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:17:07,276 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:17:07,277 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1541): 1a9e9054addbd8bfe6392d1e9f0ace31/A is initiating minor compaction (all files) 2024-11-19T11:17:07,277 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 1a9e9054addbd8bfe6392d1e9f0ace31/B is initiating minor compaction (all files) 2024-11-19T11:17:07,277 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a9e9054addbd8bfe6392d1e9f0ace31/A in TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:07,277 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/7f788c0c40e64f3f85bbc8b1436e38ec, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/f84a0cc330f441c88d140b2fac283d64, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/fbfa56ad6e6c459bafe4ec1de9fb5ab7] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp, totalSize=91.6 K 2024-11-19T11:17:07,277 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:07,277 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. files: [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/7f788c0c40e64f3f85bbc8b1436e38ec, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/f84a0cc330f441c88d140b2fac283d64, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/fbfa56ad6e6c459bafe4ec1de9fb5ab7] 2024-11-19T11:17:07,277 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a9e9054addbd8bfe6392d1e9f0ace31/B in TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:07,277 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/2f7aab2807474c1d8189592cf19ee350, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/eb8c1e591f3742dabec8abcc3b6845ea, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/487f5949c8644a5abeedf2cf8f032e92] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp, totalSize=36.1 K 2024-11-19T11:17:07,277 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f788c0c40e64f3f85bbc8b1436e38ec, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732015021478 2024-11-19T11:17:07,277 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f7aab2807474c1d8189592cf19ee350, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732015021478 2024-11-19T11:17:07,277 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting f84a0cc330f441c88d140b2fac283d64, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732015022643 2024-11-19T11:17:07,277 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting eb8c1e591f3742dabec8abcc3b6845ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732015022643 2024-11-19T11:17:07,277 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbfa56ad6e6c459bafe4ec1de9fb5ab7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732015024830 2024-11-19T11:17:07,278 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 487f5949c8644a5abeedf2cf8f032e92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732015024830 2024-11-19T11:17:07,282 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:17:07,283 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a9e9054addbd8bfe6392d1e9f0ace31#B#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:17:07,283 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111922dff707b25b44cf8a650cee8e543e1f_1a9e9054addbd8bfe6392d1e9f0ace31 store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:17:07,284 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/9a05d6c016c242c484e5dad225cc4532 is 50, key is test_row_0/B:col10/1732015025980/Put/seqid=0 2024-11-19T11:17:07,285 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111922dff707b25b44cf8a650cee8e543e1f_1a9e9054addbd8bfe6392d1e9f0ace31, store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:17:07,285 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111922dff707b25b44cf8a650cee8e543e1f_1a9e9054addbd8bfe6392d1e9f0ace31 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:17:07,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742420_1596 (size=12765) 2024-11-19T11:17:07,292 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/9a05d6c016c242c484e5dad225cc4532 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/9a05d6c016c242c484e5dad225cc4532 2024-11-19T11:17:07,296 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a9e9054addbd8bfe6392d1e9f0ace31/B of 1a9e9054addbd8bfe6392d1e9f0ace31 into 9a05d6c016c242c484e5dad225cc4532(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:17:07,296 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:17:07,296 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., storeName=1a9e9054addbd8bfe6392d1e9f0ace31/B, priority=13, startTime=1732015027275; duration=0sec 2024-11-19T11:17:07,296 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T11:17:07,296 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a9e9054addbd8bfe6392d1e9f0ace31:B 2024-11-19T11:17:07,296 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:17:07,297 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:17:07,297 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1541): 1a9e9054addbd8bfe6392d1e9f0ace31/C is initiating minor compaction (all files) 2024-11-19T11:17:07,297 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a9e9054addbd8bfe6392d1e9f0ace31/C in TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:07,297 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/37a543488cb64adca91cd0832db6b748, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/fd47b0f8a03f4c67ba2e780a44956ce1, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/8c7086403a3546d4a8cc9c121110776a] into tmpdir=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp, totalSize=36.1 K 2024-11-19T11:17:07,297 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 37a543488cb64adca91cd0832db6b748, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732015021478 2024-11-19T11:17:07,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742421_1597 (size=4469) 2024-11-19T11:17:07,297 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting fd47b0f8a03f4c67ba2e780a44956ce1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732015022643 2024-11-19T11:17:07,298 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c7086403a3546d4a8cc9c121110776a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732015024830 2024-11-19T11:17:07,304 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a9e9054addbd8bfe6392d1e9f0ace31#C#compaction#505 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:17:07,305 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/ec50428096b342ebbacc39bafff431a4 is 50, key is test_row_0/C:col10/1732015025980/Put/seqid=0 2024-11-19T11:17:07,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742422_1598 (size=12765) 2024-11-19T11:17:07,314 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/ec50428096b342ebbacc39bafff431a4 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/ec50428096b342ebbacc39bafff431a4 2024-11-19T11:17:07,318 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a9e9054addbd8bfe6392d1e9f0ace31/C of 1a9e9054addbd8bfe6392d1e9f0ace31 into ec50428096b342ebbacc39bafff431a4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:17:07,318 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:17:07,318 INFO [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., storeName=1a9e9054addbd8bfe6392d1e9f0ace31/C, priority=13, startTime=1732015027276; duration=0sec 2024-11-19T11:17:07,318 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:17:07,318 DEBUG [RS:0;16c88966c0af:35833-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a9e9054addbd8bfe6392d1e9f0ace31:C 2024-11-19T11:17:07,379 INFO [Thread-2353 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2353. 2024-11-19T11:17:07,379 DEBUG [Thread-2353 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:17:07,379 DEBUG [Thread-2353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:07,379 DEBUG [Thread-2353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:07,379 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:17:07,381 INFO [Thread-2357 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2357. 2024-11-19T11:17:07,381 DEBUG [Thread-2357 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:17:07,381 DEBUG [Thread-2357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:07,381 DEBUG [Thread-2357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:07,381 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:17:07,382 INFO [Thread-2351 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2351. 2024-11-19T11:17:07,382 DEBUG [Thread-2351 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:17:07,382 DEBUG [Thread-2351 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:07,382 DEBUG [Thread-2351 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:07,382 INFO [Thread-2359 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2359. 2024-11-19T11:17:07,382 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:17:07,382 DEBUG [Thread-2359 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:17:07,382 DEBUG [Thread-2359 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:07,382 DEBUG [Thread-2359 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:07,382 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:17:07,383 INFO [Thread-2355 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2355. 2024-11-19T11:17:07,383 DEBUG [Thread-2355 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:17:07,383 DEBUG [Thread-2355 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:07,383 DEBUG [Thread-2355 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:07,383 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:17:07,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T11:17:07,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:07,427 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 1a9e9054addbd8bfe6392d1e9f0ace31 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T11:17:07,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=A 2024-11-19T11:17:07,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:17:07,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=B 2024-11-19T11:17:07,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:17:07,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=C 2024-11-19T11:17:07,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:17:07,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111981ef9a614bbc436e8f88b2a75c6bf879_1a9e9054addbd8bfe6392d1e9f0ace31 is 50, key is test_row_0/A:col10/1732015026018/Put/seqid=0 2024-11-19T11:17:07,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742423_1599 (size=12404) 2024-11-19T11:17:07,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T11:17:07,699 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a9e9054addbd8bfe6392d1e9f0ace31#A#compaction#504 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:17:07,699 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/1a454a71da3e4cc9b5a1f4ae625a1b02 is 175, key is test_row_0/A:col10/1732015025980/Put/seqid=0 2024-11-19T11:17:07,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742424_1600 (size=31719) 2024-11-19T11:17:07,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:17:07,839 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111981ef9a614bbc436e8f88b2a75c6bf879_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111981ef9a614bbc436e8f88b2a75c6bf879_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:07,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/91e66fbb209e40959a327c47e43c6605, store: [table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:17:07,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/91e66fbb209e40959a327c47e43c6605 is 175, key is test_row_0/A:col10/1732015026018/Put/seqid=0 2024-11-19T11:17:07,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742425_1601 (size=31205) 2024-11-19T11:17:08,112 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/1a454a71da3e4cc9b5a1f4ae625a1b02 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/1a454a71da3e4cc9b5a1f4ae625a1b02 2024-11-19T11:17:08,118 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a9e9054addbd8bfe6392d1e9f0ace31/A of 1a9e9054addbd8bfe6392d1e9f0ace31 into 1a454a71da3e4cc9b5a1f4ae625a1b02(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:17:08,118 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:17:08,118 INFO [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., storeName=1a9e9054addbd8bfe6392d1e9f0ace31/A, priority=13, startTime=1732015027275; duration=0sec 2024-11-19T11:17:08,118 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:17:08,118 DEBUG [RS:0;16c88966c0af:35833-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a9e9054addbd8bfe6392d1e9f0ace31:A 2024-11-19T11:17:08,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35833 {}] regionserver.HRegion(8855): Flush requested on 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:08,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. as already flushing 2024-11-19T11:17:08,169 INFO [Thread-2342 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2342. 2024-11-19T11:17:08,169 DEBUG [Thread-2342 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:17:08,169 DEBUG [Thread-2342 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:08,169 DEBUG [Thread-2342 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:08,169 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:17:08,179 INFO [Thread-2340 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2340. 2024-11-19T11:17:08,179 DEBUG [Thread-2340 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:17:08,179 DEBUG [Thread-2340 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:08,179 DEBUG [Thread-2340 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:08,179 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:17:08,189 INFO [Thread-2344 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2344. 2024-11-19T11:17:08,189 DEBUG [Thread-2344 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:17:08,189 DEBUG [Thread-2344 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:08,189 DEBUG [Thread-2344 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:08,189 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:17:08,244 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/91e66fbb209e40959a327c47e43c6605 2024-11-19T11:17:08,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/467fbec0c6d848b8a4b2a1b1cdf07ca5 is 50, key is test_row_0/B:col10/1732015026018/Put/seqid=0 2024-11-19T11:17:08,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742426_1602 (size=12251) 2024-11-19T11:17:08,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T11:17:08,652 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/467fbec0c6d848b8a4b2a1b1cdf07ca5 2024-11-19T11:17:08,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/c8a8acd88e1f4ead9709b48dfe2ce1d9 is 50, key is test_row_0/C:col10/1732015026018/Put/seqid=0 2024-11-19T11:17:08,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742427_1603 (size=12251) 2024-11-19T11:17:09,064 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/c8a8acd88e1f4ead9709b48dfe2ce1d9 2024-11-19T11:17:09,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/91e66fbb209e40959a327c47e43c6605 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/91e66fbb209e40959a327c47e43c6605 2024-11-19T11:17:09,080 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/91e66fbb209e40959a327c47e43c6605, entries=150, sequenceid=262, filesize=30.5 K 2024-11-19T11:17:09,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/467fbec0c6d848b8a4b2a1b1cdf07ca5 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/467fbec0c6d848b8a4b2a1b1cdf07ca5 2024-11-19T11:17:09,085 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/467fbec0c6d848b8a4b2a1b1cdf07ca5, entries=150, sequenceid=262, filesize=12.0 K 2024-11-19T11:17:09,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/c8a8acd88e1f4ead9709b48dfe2ce1d9 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/c8a8acd88e1f4ead9709b48dfe2ce1d9 2024-11-19T11:17:09,089 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/c8a8acd88e1f4ead9709b48dfe2ce1d9, entries=150, sequenceid=262, filesize=12.0 K 2024-11-19T11:17:09,090 INFO [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=20.13 KB/20610 for 1a9e9054addbd8bfe6392d1e9f0ace31 in 1663ms, sequenceid=262, compaction requested=false 2024-11-19T11:17:09,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 1a9e9054addbd8bfe6392d1e9f0ace31: 2024-11-19T11:17:09,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:09,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16c88966c0af:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-19T11:17:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-19T11:17:09,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-19T11:17:09,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7330 sec 2024-11-19T11:17:09,095 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.7370 sec 2024-11-19T11:17:09,892 INFO [Thread-2346 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2346. 2024-11-19T11:17:09,892 DEBUG [Thread-2346 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:17:09,892 DEBUG [Thread-2346 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:09,892 DEBUG [Thread-2346 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:09,892 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:17:09,900 INFO [Thread-2348 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2348. 2024-11-19T11:17:09,900 DEBUG [Thread-2348 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T11:17:09,900 DEBUG [Thread-2348 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:09,900 DEBUG [Thread-2348 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:09,900 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:17:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T11:17:10,538 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:17:10,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T11:17:10,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-19T11:17:10,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-19T11:17:10,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-19T11:17:10,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-19T11:17:10,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-19T11:17:10,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T11:17:10,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3172 2024-11-19T11:17:10,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3329 2024-11-19T11:17:10,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3170 2024-11-19T11:17:10,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3176 2024-11-19T11:17:10,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3322 2024-11-19T11:17:10,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T11:17:10,538 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T11:17:10,539 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T11:17:10,539 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:10,539 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:10,539 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:17:10,539 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T11:17:10,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T11:17:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T11:17:10,542 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732015030542"}]},"ts":"1732015030542"} 2024-11-19T11:17:10,543 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T11:17:10,543 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T11:17:10,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T11:17:10,545 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a9e9054addbd8bfe6392d1e9f0ace31, UNASSIGN}] 2024-11-19T11:17:10,546 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a9e9054addbd8bfe6392d1e9f0ace31, UNASSIGN 2024-11-19T11:17:10,546 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=1a9e9054addbd8bfe6392d1e9f0ace31, regionState=CLOSING, regionLocation=16c88966c0af,35833,1732014855640 2024-11-19T11:17:10,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a9e9054addbd8bfe6392d1e9f0ace31, UNASSIGN because future has completed 2024-11-19T11:17:10,548 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T11:17:10,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640}] 2024-11-19T11:17:10,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T11:17:10,701 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(122): Close 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:10,701 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T11:17:10,701 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1722): Closing 1a9e9054addbd8bfe6392d1e9f0ace31, disabling compactions & flushes 2024-11-19T11:17:10,702 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:10,702 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:10,702 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. after waiting 0 ms 2024-11-19T11:17:10,702 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:10,702 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(2902): Flushing 1a9e9054addbd8bfe6392d1e9f0ace31 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T11:17:10,702 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=A 2024-11-19T11:17:10,702 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:17:10,702 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=B 2024-11-19T11:17:10,703 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:17:10,703 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a9e9054addbd8bfe6392d1e9f0ace31, store=C 2024-11-19T11:17:10,703 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T11:17:10,723 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199e3d43679421426e84a8dee7ea7657b6_1a9e9054addbd8bfe6392d1e9f0ace31 is 50, key is test_row_0/A:col10/1732015029898/Put/seqid=0 2024-11-19T11:17:10,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742428_1604 (size=12454) 2024-11-19T11:17:10,734 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:17:10,737 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199e3d43679421426e84a8dee7ea7657b6_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199e3d43679421426e84a8dee7ea7657b6_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:10,738 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/8558a705747b494192d9cc3cf5dfeffa, store: [table=TestAcidGuarantees family=A region=1a9e9054addbd8bfe6392d1e9f0ace31] 2024-11-19T11:17:10,738 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/8558a705747b494192d9cc3cf5dfeffa is 175, key is test_row_0/A:col10/1732015029898/Put/seqid=0 2024-11-19T11:17:10,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742429_1605 (size=31255) 2024-11-19T11:17:10,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T11:17:11,143 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/8558a705747b494192d9cc3cf5dfeffa 2024-11-19T11:17:11,155 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/81d957470e9d411f94542ab3e4d72d82 is 50, key is test_row_0/B:col10/1732015029898/Put/seqid=0 2024-11-19T11:17:11,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742430_1606 (size=12301) 2024-11-19T11:17:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T11:17:11,561 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/81d957470e9d411f94542ab3e4d72d82 2024-11-19T11:17:11,573 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/9798c8fba70e4fe8a43641082235a566 is 50, key is test_row_0/C:col10/1732015029898/Put/seqid=0 2024-11-19T11:17:11,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742431_1607 (size=12301) 2024-11-19T11:17:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T11:17:11,979 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/9798c8fba70e4fe8a43641082235a566 2024-11-19T11:17:11,989 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/A/8558a705747b494192d9cc3cf5dfeffa as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/8558a705747b494192d9cc3cf5dfeffa 2024-11-19T11:17:11,994 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/8558a705747b494192d9cc3cf5dfeffa, entries=150, sequenceid=271, filesize=30.5 K 2024-11-19T11:17:11,995 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/B/81d957470e9d411f94542ab3e4d72d82 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/81d957470e9d411f94542ab3e4d72d82 2024-11-19T11:17:11,998 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/81d957470e9d411f94542ab3e4d72d82, entries=150, sequenceid=271, filesize=12.0 K 2024-11-19T11:17:11,998 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/.tmp/C/9798c8fba70e4fe8a43641082235a566 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/9798c8fba70e4fe8a43641082235a566 2024-11-19T11:17:12,001 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/9798c8fba70e4fe8a43641082235a566, entries=150, sequenceid=271, filesize=12.0 K 2024-11-19T11:17:12,002 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1a9e9054addbd8bfe6392d1e9f0ace31 in 1300ms, sequenceid=271, compaction requested=true 2024-11-19T11:17:12,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/6f5da12a56fb49d78c27ca07dce419bb, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/99b034b73888449483edb9b5a677eb60, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/b244dab4084e4e6e8178ea434e4fd88a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/c771c2bfa6354d48b3649651888cf145, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/5a3f39b65a2742ec89a22ad6f812e95f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/e6b8ef3793014551aa48edfe8d8eb182, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/5a6688d029cf49e4a04c98a137ce9c4c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/c01dad77b5384c308d879bbae71880c5, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/4dd44284c1e64894966c7086f84bfe20, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/ab8a81300c7048cb8096df003e7d293b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/27ef460163494342b30559c7ee0330b2, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/ccebe494bc084c31898c12df4f65ff93, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/1b02a8cef68f47f5ad9b87229476719b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/640bdb92966f469ea3907bf53d6e548b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/7f788c0c40e64f3f85bbc8b1436e38ec, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/5390836a18b64fd885b4d23e10a8b1a4, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/f84a0cc330f441c88d140b2fac283d64, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/fbfa56ad6e6c459bafe4ec1de9fb5ab7] to archive 2024-11-19T11:17:12,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T11:17:12,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/6f5da12a56fb49d78c27ca07dce419bb to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/6f5da12a56fb49d78c27ca07dce419bb 2024-11-19T11:17:12,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/99b034b73888449483edb9b5a677eb60 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/99b034b73888449483edb9b5a677eb60 2024-11-19T11:17:12,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/b244dab4084e4e6e8178ea434e4fd88a to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/b244dab4084e4e6e8178ea434e4fd88a 2024-11-19T11:17:12,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/c771c2bfa6354d48b3649651888cf145 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/c771c2bfa6354d48b3649651888cf145 2024-11-19T11:17:12,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/5a3f39b65a2742ec89a22ad6f812e95f to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/5a3f39b65a2742ec89a22ad6f812e95f 2024-11-19T11:17:12,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/e6b8ef3793014551aa48edfe8d8eb182 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/e6b8ef3793014551aa48edfe8d8eb182 2024-11-19T11:17:12,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/5a6688d029cf49e4a04c98a137ce9c4c to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/5a6688d029cf49e4a04c98a137ce9c4c 2024-11-19T11:17:12,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/c01dad77b5384c308d879bbae71880c5 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/c01dad77b5384c308d879bbae71880c5 2024-11-19T11:17:12,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/4dd44284c1e64894966c7086f84bfe20 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/4dd44284c1e64894966c7086f84bfe20 2024-11-19T11:17:12,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/ab8a81300c7048cb8096df003e7d293b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/ab8a81300c7048cb8096df003e7d293b 2024-11-19T11:17:12,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/27ef460163494342b30559c7ee0330b2 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/27ef460163494342b30559c7ee0330b2 2024-11-19T11:17:12,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/ccebe494bc084c31898c12df4f65ff93 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/ccebe494bc084c31898c12df4f65ff93 2024-11-19T11:17:12,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/1b02a8cef68f47f5ad9b87229476719b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/1b02a8cef68f47f5ad9b87229476719b 2024-11-19T11:17:12,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/640bdb92966f469ea3907bf53d6e548b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/640bdb92966f469ea3907bf53d6e548b 2024-11-19T11:17:12,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/7f788c0c40e64f3f85bbc8b1436e38ec to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/7f788c0c40e64f3f85bbc8b1436e38ec 2024-11-19T11:17:12,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/5390836a18b64fd885b4d23e10a8b1a4 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/5390836a18b64fd885b4d23e10a8b1a4 2024-11-19T11:17:12,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/f84a0cc330f441c88d140b2fac283d64 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/f84a0cc330f441c88d140b2fac283d64 2024-11-19T11:17:12,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/fbfa56ad6e6c459bafe4ec1de9fb5ab7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/fbfa56ad6e6c459bafe4ec1de9fb5ab7 2024-11-19T11:17:12,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/24a2b8982fb34cbcb5334dafcce15ca7, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/26ec59464367472f8d0f4fb938965885, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/d6350042017a47a4ac62b86bb4b430e3, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/5d90cb062f5141709e2b93668a548afb, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/daf45d553f904bc7a579a54a90b6e382, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/1d10180dccc748fca1a75a6608389506, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/ca08e21157dd4502b390d7656970268a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/391a5cec20f746f3bd68b21efd396421, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/e22622a756d3438eabd540a09f291230, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/45afc29dc63e44deb7ce7288caa9bd88, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/6b51565e4c104a72a09085d99bb0d298, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/7627225f1a5443f1b5cb75db08391a23, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/04391d7d576240bba230eb582976015b, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/6a73a649b8be4c2f882c78b8080bf492, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/2f7aab2807474c1d8189592cf19ee350, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/9f0a650cff79487bb89346457ce1411a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/eb8c1e591f3742dabec8abcc3b6845ea, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/487f5949c8644a5abeedf2cf8f032e92] to archive 2024-11-19T11:17:12,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T11:17:12,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/24a2b8982fb34cbcb5334dafcce15ca7 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/24a2b8982fb34cbcb5334dafcce15ca7 2024-11-19T11:17:12,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/26ec59464367472f8d0f4fb938965885 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/26ec59464367472f8d0f4fb938965885 2024-11-19T11:17:12,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/d6350042017a47a4ac62b86bb4b430e3 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/d6350042017a47a4ac62b86bb4b430e3 2024-11-19T11:17:12,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/5d90cb062f5141709e2b93668a548afb to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/5d90cb062f5141709e2b93668a548afb 2024-11-19T11:17:12,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/daf45d553f904bc7a579a54a90b6e382 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/daf45d553f904bc7a579a54a90b6e382 2024-11-19T11:17:12,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/1d10180dccc748fca1a75a6608389506 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/1d10180dccc748fca1a75a6608389506 2024-11-19T11:17:12,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/ca08e21157dd4502b390d7656970268a to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/ca08e21157dd4502b390d7656970268a 2024-11-19T11:17:12,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/391a5cec20f746f3bd68b21efd396421 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/391a5cec20f746f3bd68b21efd396421 2024-11-19T11:17:12,032 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/e22622a756d3438eabd540a09f291230 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/e22622a756d3438eabd540a09f291230 2024-11-19T11:17:12,032 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/45afc29dc63e44deb7ce7288caa9bd88 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/45afc29dc63e44deb7ce7288caa9bd88 2024-11-19T11:17:12,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/6b51565e4c104a72a09085d99bb0d298 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/6b51565e4c104a72a09085d99bb0d298 2024-11-19T11:17:12,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/7627225f1a5443f1b5cb75db08391a23 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/7627225f1a5443f1b5cb75db08391a23 2024-11-19T11:17:12,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/04391d7d576240bba230eb582976015b to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/04391d7d576240bba230eb582976015b 2024-11-19T11:17:12,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/6a73a649b8be4c2f882c78b8080bf492 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/6a73a649b8be4c2f882c78b8080bf492 2024-11-19T11:17:12,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/2f7aab2807474c1d8189592cf19ee350 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/2f7aab2807474c1d8189592cf19ee350 2024-11-19T11:17:12,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/9f0a650cff79487bb89346457ce1411a to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/9f0a650cff79487bb89346457ce1411a 2024-11-19T11:17:12,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/eb8c1e591f3742dabec8abcc3b6845ea to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/eb8c1e591f3742dabec8abcc3b6845ea 2024-11-19T11:17:12,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/487f5949c8644a5abeedf2cf8f032e92 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/487f5949c8644a5abeedf2cf8f032e92 2024-11-19T11:17:12,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/654c49f10ec6404087109a8cb2883d4c, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/64ab895b3c724a2cbe931da876bf3aea, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/4945819ed81d4949beacc6687ea7542e, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/575d658157b84e9da3f39bbe4bbb85cc, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/8102e9060c6842fda0f0ca8e1e3ff7f8, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/b165fd7779e94f128883cdb7a84e806a, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/24299380c38c41ed8cc2d6aede367cae, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/03c8e20a43cd4ad9ab7364a9b828abe5, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/400b9a58c4c5479a927c03a04274fb49, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/bf629a0724fc493b99d5054e63aea8f1, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/2815a2e7e0ad4281b4fc9fa888f90023, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/64f83db738fb4ffd91ba1bb0b97e81ef, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/9b65d114e78c45168022d433a45e5782, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/d6a3908c3bb448fbacc2e91003e3b87f, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/37a543488cb64adca91cd0832db6b748, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/345532e00c99414f92af937897d2d203, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/fd47b0f8a03f4c67ba2e780a44956ce1, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/8c7086403a3546d4a8cc9c121110776a] to archive 2024-11-19T11:17:12,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T11:17:12,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/654c49f10ec6404087109a8cb2883d4c to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/654c49f10ec6404087109a8cb2883d4c 2024-11-19T11:17:12,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/64ab895b3c724a2cbe931da876bf3aea to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/64ab895b3c724a2cbe931da876bf3aea 2024-11-19T11:17:12,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/4945819ed81d4949beacc6687ea7542e to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/4945819ed81d4949beacc6687ea7542e 2024-11-19T11:17:12,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/575d658157b84e9da3f39bbe4bbb85cc to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/575d658157b84e9da3f39bbe4bbb85cc 2024-11-19T11:17:12,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/8102e9060c6842fda0f0ca8e1e3ff7f8 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/8102e9060c6842fda0f0ca8e1e3ff7f8 2024-11-19T11:17:12,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/b165fd7779e94f128883cdb7a84e806a to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/b165fd7779e94f128883cdb7a84e806a 2024-11-19T11:17:12,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/24299380c38c41ed8cc2d6aede367cae to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/24299380c38c41ed8cc2d6aede367cae 2024-11-19T11:17:12,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/03c8e20a43cd4ad9ab7364a9b828abe5 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/03c8e20a43cd4ad9ab7364a9b828abe5 2024-11-19T11:17:12,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/400b9a58c4c5479a927c03a04274fb49 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/400b9a58c4c5479a927c03a04274fb49 2024-11-19T11:17:12,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/bf629a0724fc493b99d5054e63aea8f1 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/bf629a0724fc493b99d5054e63aea8f1 2024-11-19T11:17:12,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/2815a2e7e0ad4281b4fc9fa888f90023 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/2815a2e7e0ad4281b4fc9fa888f90023 2024-11-19T11:17:12,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/64f83db738fb4ffd91ba1bb0b97e81ef to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/64f83db738fb4ffd91ba1bb0b97e81ef 2024-11-19T11:17:12,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/9b65d114e78c45168022d433a45e5782 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/9b65d114e78c45168022d433a45e5782 2024-11-19T11:17:12,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/d6a3908c3bb448fbacc2e91003e3b87f to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/d6a3908c3bb448fbacc2e91003e3b87f 2024-11-19T11:17:12,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/37a543488cb64adca91cd0832db6b748 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/37a543488cb64adca91cd0832db6b748 2024-11-19T11:17:12,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/345532e00c99414f92af937897d2d203 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/345532e00c99414f92af937897d2d203 2024-11-19T11:17:12,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/fd47b0f8a03f4c67ba2e780a44956ce1 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/fd47b0f8a03f4c67ba2e780a44956ce1 2024-11-19T11:17:12,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/8c7086403a3546d4a8cc9c121110776a to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/8c7086403a3546d4a8cc9c121110776a 2024-11-19T11:17:12,055 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/recovered.edits/274.seqid, newMaxSeqId=274, maxSeqId=4 2024-11-19T11:17:12,056 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. 2024-11-19T11:17:12,056 DEBUG [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1676): Region close journal for 1a9e9054addbd8bfe6392d1e9f0ace31: Waiting for close lock at 1732015030701Running coprocessor pre-close hooks at 1732015030701Disabling compacts and flushes for region at 1732015030701Disabling writes for close at 1732015030702 (+1 ms)Obtaining lock to block concurrent updates at 1732015030702Preparing flush snapshotting stores in 1a9e9054addbd8bfe6392d1e9f0ace31 at 1732015030702Finished memstore snapshotting TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732015030703 (+1 ms)Flushing stores of TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31. at 1732015030704 (+1 ms)Flushing 1a9e9054addbd8bfe6392d1e9f0ace31/A: creating writer at 1732015030704Flushing mob file 1a9e9054addbd8bfe6392d1e9f0ace31/A: appending metadata at 1732015030722 (+18 ms)Flushing mob file 1a9e9054addbd8bfe6392d1e9f0ace31/A: closing flushed file at 1732015030722Flushing 1a9e9054addbd8bfe6392d1e9f0ace31/A: appending metadata at 1732015030738 (+16 ms)Flushing 1a9e9054addbd8bfe6392d1e9f0ace31/A: closing flushed file at 1732015030738Flushing 1a9e9054addbd8bfe6392d1e9f0ace31/B: creating writer at 1732015031151 (+413 ms)Flushing 1a9e9054addbd8bfe6392d1e9f0ace31/B: appending metadata at 1732015031155 (+4 ms)Flushing 1a9e9054addbd8bfe6392d1e9f0ace31/B: closing flushed file at 1732015031155Flushing 1a9e9054addbd8bfe6392d1e9f0ace31/C: creating writer at 1732015031568 (+413 ms)Flushing 1a9e9054addbd8bfe6392d1e9f0ace31/C: appending metadata at 1732015031572 (+4 ms)Flushing 1a9e9054addbd8bfe6392d1e9f0ace31/C: closing flushed file at 1732015031572Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e2baf56: reopening flushed file at 1732015031987 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5be13a68: reopening flushed file at 1732015031994 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c941fb1: reopening flushed file at 1732015031998 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1a9e9054addbd8bfe6392d1e9f0ace31 in 1300ms, sequenceid=271, compaction requested=true at 1732015032002 (+4 ms)Writing region close event to WAL at 1732015032053 (+51 ms)Running coprocessor post-close hooks at 1732015032056 (+3 ms)Closed at 1732015032056 2024-11-19T11:17:12,058 INFO [RS_CLOSE_REGION-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(157): Closed 1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:12,059 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=1a9e9054addbd8bfe6392d1e9f0ace31, regionState=CLOSED 2024-11-19T11:17:12,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 because future has completed 2024-11-19T11:17:12,062 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-19T11:17:12,062 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseRegionProcedure 1a9e9054addbd8bfe6392d1e9f0ace31, server=16c88966c0af,35833,1732014855640 in 1.5130 sec 2024-11-19T11:17:12,064 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=162, resume processing ppid=161 2024-11-19T11:17:12,064 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, ppid=161, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a9e9054addbd8bfe6392d1e9f0ace31, UNASSIGN in 1.5170 sec 2024-11-19T11:17:12,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-19T11:17:12,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5200 sec 2024-11-19T11:17:12,066 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732015032066"}]},"ts":"1732015032066"} 2024-11-19T11:17:12,068 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T11:17:12,068 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T11:17:12,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5290 sec 2024-11-19T11:17:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T11:17:12,688 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:17:12,688 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T11:17:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:17:12,690 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:17:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T11:17:12,690 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=164, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:17:12,693 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:12,695 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A, FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B, FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C, FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/recovered.edits] 2024-11-19T11:17:12,699 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/1a454a71da3e4cc9b5a1f4ae625a1b02 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/1a454a71da3e4cc9b5a1f4ae625a1b02 2024-11-19T11:17:12,701 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/8558a705747b494192d9cc3cf5dfeffa to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/8558a705747b494192d9cc3cf5dfeffa 2024-11-19T11:17:12,702 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/91e66fbb209e40959a327c47e43c6605 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/A/91e66fbb209e40959a327c47e43c6605 2024-11-19T11:17:12,705 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/467fbec0c6d848b8a4b2a1b1cdf07ca5 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/467fbec0c6d848b8a4b2a1b1cdf07ca5 2024-11-19T11:17:12,707 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/81d957470e9d411f94542ab3e4d72d82 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/81d957470e9d411f94542ab3e4d72d82 2024-11-19T11:17:12,708 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/9a05d6c016c242c484e5dad225cc4532 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/B/9a05d6c016c242c484e5dad225cc4532 2024-11-19T11:17:12,710 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/9798c8fba70e4fe8a43641082235a566 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/9798c8fba70e4fe8a43641082235a566 2024-11-19T11:17:12,710 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/c8a8acd88e1f4ead9709b48dfe2ce1d9 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/c8a8acd88e1f4ead9709b48dfe2ce1d9 2024-11-19T11:17:12,711 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/ec50428096b342ebbacc39bafff431a4 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/C/ec50428096b342ebbacc39bafff431a4 2024-11-19T11:17:12,713 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/recovered.edits/274.seqid to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31/recovered.edits/274.seqid 2024-11-19T11:17:12,713 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/default/TestAcidGuarantees/1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:12,713 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T11:17:12,713 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T11:17:12,714 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T11:17:12,716 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111903f17c48103141e9a094dfc881df125c_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111903f17c48103141e9a094dfc881df125c_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:12,717 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111919123f2bc8c04c178afbe83c560fa4d7_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111919123f2bc8c04c178afbe83c560fa4d7_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:12,717 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111933abf6b121ce405f93e74291dc69e675_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111933abf6b121ce405f93e74291dc69e675_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:12,718 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194114cbb2839f45aeaef386a6a0b438a9_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194114cbb2839f45aeaef386a6a0b438a9_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:12,719 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111981ef9a614bbc436e8f88b2a75c6bf879_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111981ef9a614bbc436e8f88b2a75c6bf879_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:12,720 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111986d53b4ff5f8438b924f5751b3eb2688_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111986d53b4ff5f8438b924f5751b3eb2688_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:12,721 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199e3d43679421426e84a8dee7ea7657b6_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199e3d43679421426e84a8dee7ea7657b6_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:12,722 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a0d90f9a9d21437ba643751bfa4fbdf5_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a0d90f9a9d21437ba643751bfa4fbdf5_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:12,723 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ab42bc2051424f0aaa4c1a12209fd0ff_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ab42bc2051424f0aaa4c1a12209fd0ff_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:12,724 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b358cfd3b81e438e952930cab86b0eb7_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b358cfd3b81e438e952930cab86b0eb7_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:12,725 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bce7284f434e41059c0b16e5aeae1508_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bce7284f434e41059c0b16e5aeae1508_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:12,727 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ecafafaf1b9f4e28bd6530266caec5d5_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ecafafaf1b9f4e28bd6530266caec5d5_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:12,728 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ed3353ff193e463ebb51ee7d55e52e45_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ed3353ff193e463ebb51ee7d55e52e45_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:12,729 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ef411e763c1c47b995a69d74b8b06010_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ef411e763c1c47b995a69d74b8b06010_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:12,730 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f16f9e7b413c496085d5c42014941fd0_1a9e9054addbd8bfe6392d1e9f0ace31 to hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f16f9e7b413c496085d5c42014941fd0_1a9e9054addbd8bfe6392d1e9f0ace31 2024-11-19T11:17:12,731 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T11:17:12,733 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=164, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:17:12,736 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T11:17:12,738 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T11:17:12,739 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=164, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:17:12,739 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T11:17:12,740 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732015032739"}]},"ts":"9223372036854775807"} 2024-11-19T11:17:12,742 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T11:17:12,742 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1a9e9054addbd8bfe6392d1e9f0ace31, NAME => 'TestAcidGuarantees,,1732015004271.1a9e9054addbd8bfe6392d1e9f0ace31.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T11:17:12,742 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T11:17:12,742 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732015032742"}]},"ts":"9223372036854775807"} 2024-11-19T11:17:12,744 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T11:17:12,745 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=164, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T11:17:12,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 57 msec 2024-11-19T11:17:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41441 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T11:17:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T11:17:12,798 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T11:17:12,807 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250 (was 251), OpenFileDescriptor=449 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=395 (was 423), ProcessCount=11 (was 11), AvailableMemoryMB=12261 (was 11676) - AvailableMemoryMB LEAK? - 2024-11-19T11:17:12,807 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-19T11:17:12,807 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T11:17:12,807 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T11:17:12,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:12,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:12,807 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:17:12,807 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-19T11:17:12,807 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1866486918, stopped=false 2024-11-19T11:17:12,808 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=16c88966c0af,41441,1732014854424 2024-11-19T11:17:12,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T11:17:12,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10152cd49a00001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T11:17:12,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T11:17:12,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10152cd49a00001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T11:17:12,816 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T11:17:12,817 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T11:17:12,817 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35833-0x10152cd49a00001, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T11:17:12,817 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T11:17:12,817 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T11:17:12,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:12,818 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '16c88966c0af,35833,1732014855640' ***** 2024-11-19T11:17:12,818 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T11:17:12,818 INFO [RS:0;16c88966c0af:35833 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T11:17:12,819 INFO [RS:0;16c88966c0af:35833 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T11:17:12,819 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T11:17:12,819 INFO [RS:0;16c88966c0af:35833 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T11:17:12,819 INFO [RS:0;16c88966c0af:35833 {}] regionserver.HRegionServer(959): stopping server 16c88966c0af,35833,1732014855640 2024-11-19T11:17:12,819 INFO [RS:0;16c88966c0af:35833 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T11:17:12,819 INFO [RS:0;16c88966c0af:35833 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;16c88966c0af:35833. 2024-11-19T11:17:12,819 DEBUG [RS:0;16c88966c0af:35833 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T11:17:12,819 DEBUG [RS:0;16c88966c0af:35833 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:12,819 INFO [RS:0;16c88966c0af:35833 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T11:17:12,819 INFO [RS:0;16c88966c0af:35833 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T11:17:12,819 INFO [RS:0;16c88966c0af:35833 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T11:17:12,819 INFO [RS:0;16c88966c0af:35833 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T11:17:12,820 INFO [RS:0;16c88966c0af:35833 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-19T11:17:12,820 DEBUG [RS:0;16c88966c0af:35833 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-19T11:17:12,821 DEBUG [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T11:17:12,821 DEBUG [RS:0;16c88966c0af:35833 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T11:17:12,821 INFO [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T11:17:12,821 DEBUG [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T11:17:12,821 DEBUG [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T11:17:12,821 DEBUG [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T11:17:12,821 INFO [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-19T11:17:12,842 DEBUG [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740/.tmp/info/20d305e25eb642299712913e819fa750 is 95, key is TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f./info:/1732014886531/DeleteFamily/seqid=0 2024-11-19T11:17:12,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742432_1608 (size=6514) 2024-11-19T11:17:12,850 INFO [regionserver/16c88966c0af:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T11:17:13,021 DEBUG [RS:0;16c88966c0af:35833 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T11:17:13,221 DEBUG [RS:0;16c88966c0af:35833 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T11:17:13,246 INFO [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740/.tmp/info/20d305e25eb642299712913e819fa750 2024-11-19T11:17:13,266 DEBUG [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740/.tmp/ns/c01e074a5f2649aea23b4e8aa87aac74 is 93, key is TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f./ns:/1732014886468/DeleteFamily/seqid=0 2024-11-19T11:17:13,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742433_1609 (size=5906) 2024-11-19T11:17:13,421 DEBUG [RS:0;16c88966c0af:35833 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T11:17:13,521 INFO [regionserver/16c88966c0af:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-19T11:17:13,522 INFO [regionserver/16c88966c0af:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-19T11:17:13,567 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T11:17:13,622 DEBUG [RS:0;16c88966c0af:35833 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T11:17:13,669 INFO [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740/.tmp/ns/c01e074a5f2649aea23b4e8aa87aac74 2024-11-19T11:17:13,687 DEBUG [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740/.tmp/rep_barrier/4283311e038f4594b4b5a5425338c997 is 102, key is TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f./rep_barrier:/1732014886468/DeleteFamily/seqid=0 2024-11-19T11:17:13,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742434_1610 (size=6025) 2024-11-19T11:17:13,692 INFO [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740/.tmp/rep_barrier/4283311e038f4594b4b5a5425338c997 2024-11-19T11:17:13,716 DEBUG [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740/.tmp/table/b9b7e2221fcf439098ab71d19cc6691c is 96, key is TestAcidGuarantees,,1732014859171.9cbe0c20892be91e1583baaaf6ed665f./table:/1732014886468/DeleteFamily/seqid=0 2024-11-19T11:17:13,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742435_1611 (size=5936) 2024-11-19T11:17:13,822 DEBUG [RS:0;16c88966c0af:35833 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T11:17:14,022 DEBUG [RS:0;16c88966c0af:35833 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T11:17:14,120 INFO [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740/.tmp/table/b9b7e2221fcf439098ab71d19cc6691c 2024-11-19T11:17:14,131 DEBUG [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740/.tmp/info/20d305e25eb642299712913e819fa750 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740/info/20d305e25eb642299712913e819fa750 2024-11-19T11:17:14,135 INFO [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740/info/20d305e25eb642299712913e819fa750, entries=12, sequenceid=90, filesize=6.4 K 2024-11-19T11:17:14,136 DEBUG [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740/.tmp/ns/c01e074a5f2649aea23b4e8aa87aac74 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740/ns/c01e074a5f2649aea23b4e8aa87aac74 2024-11-19T11:17:14,139 INFO [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740/ns/c01e074a5f2649aea23b4e8aa87aac74, entries=8, sequenceid=90, filesize=5.8 K 2024-11-19T11:17:14,140 DEBUG [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740/.tmp/rep_barrier/4283311e038f4594b4b5a5425338c997 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740/rep_barrier/4283311e038f4594b4b5a5425338c997 2024-11-19T11:17:14,144 INFO [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740/rep_barrier/4283311e038f4594b4b5a5425338c997, entries=6, sequenceid=90, filesize=5.9 K 2024-11-19T11:17:14,145 DEBUG [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740/.tmp/table/b9b7e2221fcf439098ab71d19cc6691c as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740/table/b9b7e2221fcf439098ab71d19cc6691c 2024-11-19T11:17:14,148 INFO [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740/table/b9b7e2221fcf439098ab71d19cc6691c, entries=7, sequenceid=90, filesize=5.8 K 2024-11-19T11:17:14,149 INFO [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1328ms, sequenceid=90, compaction requested=false 2024-11-19T11:17:14,154 DEBUG [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-19T11:17:14,155 DEBUG [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T11:17:14,155 INFO [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T11:17:14,155 DEBUG [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732015032820Running coprocessor pre-close hooks at 1732015032821 (+1 ms)Disabling compacts and flushes for region at 1732015032821Disabling writes for close at 1732015032821Obtaining lock to block concurrent updates at 1732015032821Preparing flush snapshotting stores in 1588230740 at 1732015032821Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732015032822 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732015032822Flushing 1588230740/info: creating writer at 1732015032823 (+1 ms)Flushing 1588230740/info: appending metadata at 1732015032842 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732015032842Flushing 1588230740/ns: creating writer at 1732015033249 (+407 ms)Flushing 1588230740/ns: appending metadata at 1732015033265 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732015033265Flushing 1588230740/rep_barrier: creating writer at 1732015033673 (+408 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732015033687 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732015033687Flushing 1588230740/table: creating writer at 1732015033695 (+8 ms)Flushing 1588230740/table: appending metadata at 1732015033715 (+20 ms)Flushing 1588230740/table: closing flushed file at 1732015033716 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7763ead0: reopening flushed file at 1732015034129 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@537cbf21: reopening flushed file at 1732015034135 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f5d6641: reopening flushed file at 1732015034139 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72092ac7: reopening flushed file at 1732015034144 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1328ms, sequenceid=90, compaction requested=false at 1732015034149 (+5 ms)Writing region close event to WAL at 1732015034151 (+2 ms)Running coprocessor post-close hooks at 1732015034155 (+4 ms)Closed at 1732015034155 2024-11-19T11:17:14,155 DEBUG [RS_CLOSE_META-regionserver/16c88966c0af:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T11:17:14,222 INFO [RS:0;16c88966c0af:35833 {}] regionserver.HRegionServer(976): stopping server 16c88966c0af,35833,1732014855640; all regions closed. 2024-11-19T11:17:14,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741834_1010 (size=25344) 2024-11-19T11:17:14,230 DEBUG [RS:0;16c88966c0af:35833 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/oldWALs 2024-11-19T11:17:14,230 INFO [RS:0;16c88966c0af:35833 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 16c88966c0af%2C35833%2C1732014855640.meta:.meta(num 1732014858683) 2024-11-19T11:17:14,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741832_1008 (size=17332169) 2024-11-19T11:17:14,235 DEBUG [RS:0;16c88966c0af:35833 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/oldWALs 2024-11-19T11:17:14,235 INFO [RS:0;16c88966c0af:35833 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 16c88966c0af%2C35833%2C1732014855640:(num 1732014857593) 2024-11-19T11:17:14,235 DEBUG [RS:0;16c88966c0af:35833 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:17:14,235 INFO [RS:0;16c88966c0af:35833 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T11:17:14,235 INFO [RS:0;16c88966c0af:35833 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T11:17:14,235 INFO [RS:0;16c88966c0af:35833 {}] hbase.ChoreService(370): Chore service for: regionserver/16c88966c0af:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-19T11:17:14,235 INFO [RS:0;16c88966c0af:35833 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T11:17:14,235 INFO [regionserver/16c88966c0af:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T11:17:14,236 INFO [RS:0;16c88966c0af:35833 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35833 2024-11-19T11:17:14,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10152cd49a00001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/16c88966c0af,35833,1732014855640 2024-11-19T11:17:14,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T11:17:14,247 INFO [RS:0;16c88966c0af:35833 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T11:17:14,258 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [16c88966c0af,35833,1732014855640] 2024-11-19T11:17:14,268 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/16c88966c0af,35833,1732014855640 already deleted, retry=false 2024-11-19T11:17:14,268 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 16c88966c0af,35833,1732014855640 expired; onlineServers=0 2024-11-19T11:17:14,269 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '16c88966c0af,41441,1732014854424' ***** 2024-11-19T11:17:14,269 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-19T11:17:14,269 INFO [M:0;16c88966c0af:41441 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T11:17:14,269 INFO [M:0;16c88966c0af:41441 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T11:17:14,269 DEBUG [M:0;16c88966c0af:41441 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-19T11:17:14,269 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-19T11:17:14,270 DEBUG [M:0;16c88966c0af:41441 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-19T11:17:14,270 DEBUG [master/16c88966c0af:0:becomeActiveMaster-HFileCleaner.large.0-1732014857299 {}] cleaner.HFileCleaner(306): Exit Thread[master/16c88966c0af:0:becomeActiveMaster-HFileCleaner.large.0-1732014857299,5,FailOnTimeoutGroup] 2024-11-19T11:17:14,270 DEBUG [master/16c88966c0af:0:becomeActiveMaster-HFileCleaner.small.0-1732014857299 {}] cleaner.HFileCleaner(306): Exit Thread[master/16c88966c0af:0:becomeActiveMaster-HFileCleaner.small.0-1732014857299,5,FailOnTimeoutGroup] 2024-11-19T11:17:14,270 INFO [M:0;16c88966c0af:41441 {}] hbase.ChoreService(370): Chore service for: master/16c88966c0af:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-19T11:17:14,270 INFO [M:0;16c88966c0af:41441 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T11:17:14,270 DEBUG [M:0;16c88966c0af:41441 {}] master.HMaster(1795): Stopping service threads 2024-11-19T11:17:14,271 INFO [M:0;16c88966c0af:41441 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-19T11:17:14,271 INFO [M:0;16c88966c0af:41441 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T11:17:14,271 ERROR [M:0;16c88966c0af:41441 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1228365231) connection to localhost/127.0.0.1:37775 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:37775,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-19T11:17:14,273 INFO [M:0;16c88966c0af:41441 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T11:17:14,273 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-19T11:17:14,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-19T11:17:14,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T11:17:14,279 DEBUG [M:0;16c88966c0af:41441 {}] zookeeper.ZKUtil(347): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-19T11:17:14,279 WARN [M:0;16c88966c0af:41441 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-19T11:17:14,280 INFO [M:0;16c88966c0af:41441 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/.lastflushedseqids 2024-11-19T11:17:14,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742436_1612 (size=72) 2024-11-19T11:17:14,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10152cd49a00001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T11:17:14,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35833-0x10152cd49a00001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T11:17:14,360 INFO [RS:0;16c88966c0af:35833 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T11:17:14,360 INFO [RS:0;16c88966c0af:35833 {}] regionserver.HRegionServer(1031): Exiting; stopping=16c88966c0af,35833,1732014855640; zookeeper connection closed. 2024-11-19T11:17:14,361 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@148ebd67 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@148ebd67 2024-11-19T11:17:14,361 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-19T11:17:14,689 INFO [M:0;16c88966c0af:41441 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T11:17:14,689 INFO [M:0;16c88966c0af:41441 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-19T11:17:14,689 DEBUG [M:0;16c88966c0af:41441 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T11:17:14,689 INFO [M:0;16c88966c0af:41441 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T11:17:14,690 DEBUG [M:0;16c88966c0af:41441 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T11:17:14,690 DEBUG [M:0;16c88966c0af:41441 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T11:17:14,690 DEBUG [M:0;16c88966c0af:41441 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T11:17:14,690 INFO [M:0;16c88966c0af:41441 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=760.77 KB heapSize=930.51 KB 2024-11-19T11:17:14,710 DEBUG [M:0;16c88966c0af:41441 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1b20130d47134dccb7c74d5648cae066 is 82, key is hbase:meta,,1/info:regioninfo/1732014858766/Put/seqid=0 2024-11-19T11:17:14,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742437_1613 (size=5672) 2024-11-19T11:17:14,759 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T11:17:14,759 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-19T11:17:14,760 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T11:17:15,114 INFO [M:0;16c88966c0af:41441 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2096 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1b20130d47134dccb7c74d5648cae066 2024-11-19T11:17:15,143 DEBUG [M:0;16c88966c0af:41441 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3ccc71e78c4348edb8841418f505764f is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x8D/proc:d/1732015007349/Put/seqid=0 2024-11-19T11:17:15,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742438_1614 (size=44306) 2024-11-19T11:17:15,547 INFO [M:0;16c88966c0af:41441 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=760.21 KB at sequenceid=2096 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3ccc71e78c4348edb8841418f505764f 2024-11-19T11:17:15,552 INFO [M:0;16c88966c0af:41441 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3ccc71e78c4348edb8841418f505764f 2024-11-19T11:17:15,570 DEBUG [M:0;16c88966c0af:41441 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/aba7c755e36c4687be83fe85143dfe64 is 69, key is 16c88966c0af,35833,1732014855640/rs:state/1732014857347/Put/seqid=0 2024-11-19T11:17:15,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073742439_1615 (size=5156) 2024-11-19T11:17:15,974 INFO [M:0;16c88966c0af:41441 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2096 (bloomFilter=true), to=hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/aba7c755e36c4687be83fe85143dfe64 2024-11-19T11:17:15,977 DEBUG [M:0;16c88966c0af:41441 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1b20130d47134dccb7c74d5648cae066 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1b20130d47134dccb7c74d5648cae066 2024-11-19T11:17:15,980 INFO [M:0;16c88966c0af:41441 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1b20130d47134dccb7c74d5648cae066, entries=8, sequenceid=2096, filesize=5.5 K 2024-11-19T11:17:15,981 DEBUG [M:0;16c88966c0af:41441 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3ccc71e78c4348edb8841418f505764f as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3ccc71e78c4348edb8841418f505764f 2024-11-19T11:17:15,983 INFO [M:0;16c88966c0af:41441 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3ccc71e78c4348edb8841418f505764f 2024-11-19T11:17:15,983 INFO [M:0;16c88966c0af:41441 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3ccc71e78c4348edb8841418f505764f, entries=164, sequenceid=2096, filesize=43.3 K 2024-11-19T11:17:15,984 DEBUG [M:0;16c88966c0af:41441 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/aba7c755e36c4687be83fe85143dfe64 as hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/aba7c755e36c4687be83fe85143dfe64 2024-11-19T11:17:15,986 INFO [M:0;16c88966c0af:41441 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37775/user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/aba7c755e36c4687be83fe85143dfe64, entries=1, sequenceid=2096, filesize=5.0 K 2024-11-19T11:17:15,987 INFO [M:0;16c88966c0af:41441 {}] regionserver.HRegion(3140): Finished flush of dataSize ~760.77 KB/779027, heapSize ~930.21 KB/952536, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1297ms, sequenceid=2096, compaction requested=false 2024-11-19T11:17:15,988 INFO [M:0;16c88966c0af:41441 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T11:17:15,988 DEBUG [M:0;16c88966c0af:41441 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732015034689Disabling compacts and flushes for region at 1732015034689Disabling writes for close at 1732015034690 (+1 ms)Obtaining lock to block concurrent updates at 1732015034690Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732015034690Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=779027, getHeapSize=952776, getOffHeapSize=0, getCellsCount=2285 at 1732015034690Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732015034691 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732015034691Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732015034709 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732015034709Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732015035123 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732015035143 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732015035143Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732015035552 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732015035570 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732015035570Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34c3dbca: reopening flushed file at 1732015035977 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1acf5541: reopening flushed file at 1732015035980 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a2022ba: reopening flushed file at 1732015035983 (+3 ms)Finished flush of dataSize ~760.77 KB/779027, heapSize ~930.21 KB/952536, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1297ms, sequenceid=2096, compaction requested=false at 1732015035987 (+4 ms)Writing region close event to WAL at 1732015035988 (+1 ms)Closed at 1732015035988 2024-11-19T11:17:15,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42293 is added to blk_1073741830_1006 (size=916442) 2024-11-19T11:17:15,991 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c1f1ca8a-a87d-2b13-d6a1-3306e5bb7094/MasterData/WALs/16c88966c0af,41441,1732014854424/16c88966c0af%2C41441%2C1732014854424.1732014856621 not finished, retry = 0 2024-11-19T11:17:16,092 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T11:17:16,092 INFO [M:0;16c88966c0af:41441 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-19T11:17:16,092 INFO [M:0;16c88966c0af:41441 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41441 2024-11-19T11:17:16,092 INFO [M:0;16c88966c0af:41441 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T11:17:16,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T11:17:16,245 INFO [M:0;16c88966c0af:41441 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T11:17:16,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41441-0x10152cd49a00000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T11:17:16,254 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c68b1bf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T11:17:16,257 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T11:17:16,257 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T11:17:16,258 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T11:17:16,258 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/hadoop.log.dir/,STOPPED} 2024-11-19T11:17:16,260 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-19T11:17:16,260 WARN [BP-1532712323-172.17.0.2-1732014850042 heartbeating to localhost/127.0.0.1:37775 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-19T11:17:16,260 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-19T11:17:16,260 WARN [BP-1532712323-172.17.0.2-1732014850042 heartbeating to localhost/127.0.0.1:37775 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1532712323-172.17.0.2-1732014850042 (Datanode Uuid 9754eb23-22ad-4701-a420-3678696778b6) service to localhost/127.0.0.1:37775 2024-11-19T11:17:16,263 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/cluster_5de41c1e-6874-2d94-bb17-966f646923cf/data/data1/current/BP-1532712323-172.17.0.2-1732014850042 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T11:17:16,263 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/cluster_5de41c1e-6874-2d94-bb17-966f646923cf/data/data2/current/BP-1532712323-172.17.0.2-1732014850042 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T11:17:16,263 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-19T11:17:16,270 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T11:17:16,270 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T11:17:16,271 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T11:17:16,271 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T11:17:16,271 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b5cb3c4-35a0-8595-f81f-72a1e364e44b/hadoop.log.dir/,STOPPED} 2024-11-19T11:17:16,285 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-19T11:17:16,425 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down